@crawlee/core 4.0.0-beta.17 → 4.0.0-beta.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/http_clients/base-http-client.d.ts +19 -68
- package/http_clients/base-http-client.d.ts.map +1 -1
- package/http_clients/base-http-client.js +13 -9
- package/http_clients/base-http-client.js.map +1 -1
- package/http_clients/got-scraping-http-client.d.ts +3 -3
- package/http_clients/got-scraping-http-client.d.ts.map +1 -1
- package/http_clients/got-scraping-http-client.js +27 -10
- package/http_clients/got-scraping-http-client.js.map +1 -1
- package/package.json +5 -5
|
@@ -1,30 +1,7 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { Readable } from 'node:stream';
|
|
2
2
|
import type { AllowedHttpMethods } from '@crawlee/types';
|
|
3
3
|
import { type SearchParams } from '@crawlee/utils';
|
|
4
|
-
import type {
|
|
5
|
-
type Timeout = {
|
|
6
|
-
lookup: number;
|
|
7
|
-
connect: number;
|
|
8
|
-
secureConnect: number;
|
|
9
|
-
socket: number;
|
|
10
|
-
send: number;
|
|
11
|
-
response: number;
|
|
12
|
-
} | {
|
|
13
|
-
request: number;
|
|
14
|
-
};
|
|
15
|
-
/**
|
|
16
|
-
* Maps permitted values of the `responseType` option on {@link HttpRequest} to the types that they produce.
|
|
17
|
-
*/
|
|
18
|
-
export interface ResponseTypes {
|
|
19
|
-
'json': unknown;
|
|
20
|
-
'text': string;
|
|
21
|
-
'buffer': Buffer;
|
|
22
|
-
}
|
|
23
|
-
interface Progress {
|
|
24
|
-
percent: number;
|
|
25
|
-
transferred: number;
|
|
26
|
-
total?: number;
|
|
27
|
-
}
|
|
4
|
+
import type { Session } from '../session_pool/session.js';
|
|
28
5
|
interface ToughCookieJar {
|
|
29
6
|
getCookieString: ((currentUrl: string, options: Record<string, unknown>, callback: (error: Error | null, cookies: string) => void) => string) & ((url: string, callback: (error: Error | null, cookieHeader: string) => void) => string);
|
|
30
7
|
setCookie: ((cookieOrString: unknown, currentUrl: string, options: Record<string, unknown>, callback: (error: Error | null, cookie: unknown) => void) => void) & ((rawCookie: string, url: string, callback: (error: Error | null, result: unknown) => void) => void);
|
|
@@ -33,23 +10,20 @@ interface PromiseCookieJar {
|
|
|
33
10
|
getCookieString: (url: string) => Promise<string>;
|
|
34
11
|
setCookie: (rawCookie: string, url: string) => Promise<unknown>;
|
|
35
12
|
}
|
|
36
|
-
type SimpleHeaders = Record<string, string | string[] | undefined>;
|
|
37
13
|
/**
|
|
38
14
|
* HTTP Request as accepted by {@link BaseHttpClient} methods.
|
|
39
15
|
*/
|
|
40
|
-
export interface HttpRequest
|
|
41
|
-
[k: string]: unknown;
|
|
16
|
+
export interface HttpRequest {
|
|
42
17
|
url: string | URL;
|
|
43
18
|
method?: AllowedHttpMethods;
|
|
44
|
-
headers?:
|
|
45
|
-
body?:
|
|
19
|
+
headers?: Headers;
|
|
20
|
+
body?: Readable;
|
|
46
21
|
signal?: AbortSignal;
|
|
47
|
-
timeout?:
|
|
22
|
+
timeout?: number;
|
|
48
23
|
cookieJar?: ToughCookieJar | PromiseCookieJar;
|
|
49
24
|
followRedirect?: boolean | ((response: any) => boolean);
|
|
50
25
|
maxRedirects?: number;
|
|
51
26
|
encoding?: BufferEncoding;
|
|
52
|
-
responseType?: TResponseType;
|
|
53
27
|
throwHttpErrors?: boolean;
|
|
54
28
|
proxyUrl?: string;
|
|
55
29
|
headerGeneratorOptions?: Record<string, unknown>;
|
|
@@ -63,7 +37,7 @@ export interface HttpRequest<TResponseType extends keyof ResponseTypes = 'text'>
|
|
|
63
37
|
/**
|
|
64
38
|
* Additional options for HTTP requests that need to be handled separately before passing to {@link BaseHttpClient}.
|
|
65
39
|
*/
|
|
66
|
-
export interface HttpRequestOptions
|
|
40
|
+
export interface HttpRequestOptions extends HttpRequest {
|
|
67
41
|
/** Search (query string) parameters to be appended to the request URL */
|
|
68
42
|
searchParams?: SearchParams;
|
|
69
43
|
/** A form to be sent in the HTTP request body (URL encoding will be used) */
|
|
@@ -75,50 +49,27 @@ export interface HttpRequestOptions<TResponseType extends keyof ResponseTypes =
|
|
|
75
49
|
/** Basic HTTP Auth password */
|
|
76
50
|
password?: string;
|
|
77
51
|
}
|
|
78
|
-
/**
|
|
79
|
-
* HTTP response data, without a body, as returned by {@link BaseHttpClient} methods.
|
|
80
|
-
*/
|
|
81
|
-
export interface BaseHttpResponseData {
|
|
82
|
-
redirectUrls: URL[];
|
|
83
|
-
url: string;
|
|
84
|
-
ip?: string;
|
|
85
|
-
statusCode: number;
|
|
86
|
-
statusMessage?: string;
|
|
87
|
-
headers: SimpleHeaders;
|
|
88
|
-
trailers: SimpleHeaders;
|
|
89
|
-
complete: boolean;
|
|
90
|
-
}
|
|
91
|
-
interface HttpResponseWithoutBody<TResponseType extends keyof ResponseTypes = keyof ResponseTypes> extends BaseHttpResponseData {
|
|
92
|
-
request: HttpRequest<TResponseType>;
|
|
93
|
-
}
|
|
94
52
|
export declare class ResponseWithUrl extends Response {
|
|
95
53
|
url: string;
|
|
96
54
|
constructor(body: BodyInit | null, init: ResponseInit & {
|
|
97
55
|
url?: string;
|
|
98
56
|
});
|
|
99
57
|
}
|
|
100
|
-
/**
|
|
101
|
-
* HTTP response data as returned by the {@link BaseHttpClient.sendRequest} method.
|
|
102
|
-
*/
|
|
103
|
-
export interface HttpResponse<TResponseType extends keyof ResponseTypes = keyof ResponseTypes> extends HttpResponseWithoutBody<TResponseType> {
|
|
104
|
-
[k: string]: any;
|
|
105
|
-
body: ResponseTypes[TResponseType];
|
|
106
|
-
}
|
|
107
|
-
/**
|
|
108
|
-
* HTTP response data as returned by the {@link BaseHttpClient.stream} method.
|
|
109
|
-
*/
|
|
110
|
-
export interface StreamingHttpResponse extends HttpResponseWithoutBody {
|
|
111
|
-
stream: Readable;
|
|
112
|
-
readonly downloadProgress: Progress;
|
|
113
|
-
readonly uploadProgress: Progress;
|
|
114
|
-
}
|
|
115
58
|
/**
|
|
116
59
|
* Type of a function called when an HTTP redirect takes place. It is allowed to mutate the `updatedRequest` argument.
|
|
117
60
|
*/
|
|
118
61
|
export type RedirectHandler = (redirectResponse: Response, updatedRequest: {
|
|
119
62
|
url?: string | URL;
|
|
120
|
-
headers:
|
|
63
|
+
headers: Headers;
|
|
121
64
|
}) => void;
|
|
65
|
+
export interface SendRequestOptions {
|
|
66
|
+
session?: Session;
|
|
67
|
+
cookieJar?: ToughCookieJar;
|
|
68
|
+
timeout?: number;
|
|
69
|
+
}
|
|
70
|
+
export interface StreamOptions extends SendRequestOptions {
|
|
71
|
+
onRedirect?: RedirectHandler;
|
|
72
|
+
}
|
|
122
73
|
/**
|
|
123
74
|
* Interface for user-defined HTTP clients to be used for plain HTTP crawling and for sending additional requests during a crawl.
|
|
124
75
|
*/
|
|
@@ -126,15 +77,15 @@ export interface BaseHttpClient {
|
|
|
126
77
|
/**
|
|
127
78
|
* Perform an HTTP Request and return the complete response.
|
|
128
79
|
*/
|
|
129
|
-
sendRequest
|
|
80
|
+
sendRequest(request: Request, options?: SendRequestOptions): Promise<Response>;
|
|
130
81
|
/**
|
|
131
82
|
* Perform an HTTP Request and return after the response headers are received. The body may be read from a stream contained in the response.
|
|
132
83
|
*/
|
|
133
|
-
stream(request:
|
|
84
|
+
stream(request: Request, options?: StreamOptions): Promise<Response>;
|
|
134
85
|
}
|
|
135
86
|
/**
|
|
136
87
|
* Converts {@link HttpRequestOptions} to a {@link HttpRequest}.
|
|
137
88
|
*/
|
|
138
|
-
export declare function processHttpRequestOptions
|
|
89
|
+
export declare function processHttpRequestOptions({ searchParams, form, json, username, password, ...request }: HttpRequestOptions): HttpRequest;
|
|
139
90
|
export {};
|
|
140
91
|
//# sourceMappingURL=base-http-client.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"base-http-client.d.ts","sourceRoot":"","sources":["../../src/http_clients/base-http-client.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"base-http-client.d.ts","sourceRoot":"","sources":["../../src/http_clients/base-http-client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEvC,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AACzD,OAAO,EAAqB,KAAK,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAEtE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAG1D,UAAU,cAAc;IACpB,eAAe,EAAE,CAAC,CACd,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAChC,QAAQ,EAAE,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,KACvD,MAAM,CAAC,GACR,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,EAAE,YAAY,EAAE,MAAM,KAAK,IAAI,KAAK,MAAM,CAAC,CAAC;IAC7F,SAAS,EAAE,CAAC,CACR,cAAc,EAAE,OAAO,EACvB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAChC,QAAQ,EAAE,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,EAAE,MAAM,EAAE,OAAO,KAAK,IAAI,KACvD,IAAI,CAAC,GACN,CAAC,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,EAAE,MAAM,EAAE,OAAO,KAAK,IAAI,KAAK,IAAI,CAAC,CAAC;CAC5G;AAED,UAAU,gBAAgB;IACtB,eAAe,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IAClD,SAAS,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;CACnE;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IACxB,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;IAClB,MAAM,CAAC,EAAE,kBAAkB,CAAC;IAC5B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,IAAI,CAAC,EAAE,QAAQ,CAAC;IAEhB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,SAAS,CAAC,EAAE,cAAc,GAAG,gBAAgB,CAAC;IAC9C,cAAc,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,QAAQ,EAAE,GAAG,KAAK,OAAO,CAAC,CAAC;IACxD,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB,QAAQ,CAAC,EAAE,cAAc,CAAC;IAC1B,eAAe,CAAC,EAAE,OAAO,CAAC;IAG1B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sBAAsB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjD,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,eAAe,CAAC,EAAE;QACd,UAAU,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAC5E,CAAC;IACF,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAmB,SAAQ,WAAW;IACnD,yEAAyE;IACzE,YAAY,CAAC,EAAE,YAAY,CAAC;IAE5B,6EAA6E;IAC7E,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9B,gFAAgF;IAChF,IAAI,CAAC,EAAE,OAAO,CAAC;IAEf,+BAA+B;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,+BAA+B;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,qBAAa,eAAgB,SAAQ,QAAQ;IAChC,GAAG,EAAE,MAAM,CAAC;gBACT,IAAI,EAAE,QAAQ,GAAG,IAAI,EAAE,IAAI,EAAE,YAAY,GAAG;QAAE,GAAG,CAAC,EAAE,MAAM,CAAA;KAAE;CAI3E;AAED;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,CAC1B,gBAAgB,EAAE,QAAQ,EAC1B,cAAc,EAAE;IAAE,GAAG,CAAC,EAAE,MAAM,GAAG,GAAG,CAAC;IAAC,OAAO,EAAE,OAAO,CAAA;CAAE,KACvD,IAAI,CAAC;AAEV,MAAM,WAAW,kBAAkB;IAC/B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,SAAS,CAAC,EAAE,cAAc,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,aAAc,SAAQ,kBAAkB;IACrD,UAAU,CAAC,EAAE,eAAe,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC3B;;OAEG;IACH,WAAW,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IAE/E;;OAEG;IACH,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;CACxE;AAED;;GAEG;AACH,wBAAgB,yBAAyB,CAAC,EACtC,YAAY,EACZ,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,GAAG,OAAO,EACb,EAAE,kBAAkB,GAAG,WAAW,CAwClC"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { Readable } from 'node:stream';
|
|
1
2
|
import { applySearchParams } from '@crawlee/utils';
|
|
2
3
|
export class ResponseWithUrl extends Response {
|
|
3
4
|
url;
|
|
@@ -11,29 +12,32 @@ export class ResponseWithUrl extends Response {
|
|
|
11
12
|
*/
|
|
12
13
|
export function processHttpRequestOptions({ searchParams, form, json, username, password, ...request }) {
|
|
13
14
|
const url = new URL(request.url);
|
|
14
|
-
const headers =
|
|
15
|
+
const headers = new Headers(request.headers);
|
|
15
16
|
applySearchParams(url, searchParams);
|
|
16
17
|
if ([request.body, form, json].filter((value) => value !== undefined).length > 1) {
|
|
17
18
|
throw new Error('At most one of `body`, `form` and `json` may be specified in sendRequest arguments');
|
|
18
19
|
}
|
|
19
20
|
const body = (() => {
|
|
20
21
|
if (form !== undefined) {
|
|
21
|
-
return new URLSearchParams(form).toString();
|
|
22
|
+
return Readable.from(new URLSearchParams(form).toString());
|
|
22
23
|
}
|
|
23
24
|
if (json !== undefined) {
|
|
24
|
-
return JSON.stringify(json);
|
|
25
|
+
return Readable.from(JSON.stringify(json));
|
|
25
26
|
}
|
|
26
|
-
|
|
27
|
+
if (request.body !== undefined) {
|
|
28
|
+
return Readable.from(request.body);
|
|
29
|
+
}
|
|
30
|
+
return undefined;
|
|
27
31
|
})();
|
|
28
|
-
if (form !== undefined) {
|
|
29
|
-
headers
|
|
32
|
+
if (form !== undefined && !headers.has('content-type')) {
|
|
33
|
+
headers.set('content-type', 'application/x-www-form-urlencoded');
|
|
30
34
|
}
|
|
31
|
-
if (json !== undefined) {
|
|
32
|
-
headers
|
|
35
|
+
if (json !== undefined && !headers.has('content-type')) {
|
|
36
|
+
headers.set('content-type', 'application/json');
|
|
33
37
|
}
|
|
34
38
|
if (username !== undefined || password !== undefined) {
|
|
35
39
|
const encodedAuth = Buffer.from(`${username ?? ''}:${password ?? ''}`).toString('base64');
|
|
36
|
-
headers.authorization
|
|
40
|
+
headers.set('authorization', `Basic ${encodedAuth}`);
|
|
37
41
|
}
|
|
38
42
|
return { ...request, body, url, headers };
|
|
39
43
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"base-http-client.js","sourceRoot":"","sources":["../../src/http_clients/base-http-client.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"base-http-client.js","sourceRoot":"","sources":["../../src/http_clients/base-http-client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAGvC,OAAO,EAAE,iBAAiB,EAAqB,MAAM,gBAAgB,CAAC;AA0EtE,MAAM,OAAO,eAAgB,SAAQ,QAAQ;IAChC,GAAG,CAAS;IACrB,YAAY,IAAqB,EAAE,IAAqC;QACpE,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAClB,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC;IAC9B,CAAC;CACJ;AAmCD;;GAEG;AACH,MAAM,UAAU,yBAAyB,CAAC,EACtC,YAAY,EACZ,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,GAAG,OAAO,EACO;IACjB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACjC,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAE7C,iBAAiB,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IAErC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/E,MAAM,IAAI,KAAK,CAAC,oFAAoF,CAAC,CAAC;IAC1G,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,GAAG,EAAE;QACf,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACrB,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC/D,CAAC;QAED,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACrB,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/C,CAAC;QAED,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC7B,OAAO,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACvC,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC,CAAC,EAAE,CAAC;IAEL,IAAI,IAAI,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,mCAAmC,CAAC,CAAC;IACrE,CAAC;IAED,IAAI,IAAI,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,kBAAkB,CAAC,CAAC;IACpD,CAAC;IAED,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QACnD,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,IAAI,EAAE,IAAI,QAAQ,IAAI,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC1F,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,SAAS,WAAW,EAAE,CAAC,CAAC;IACzD,CAAC;IAED,OAAO,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC;AAC9C,CAAC"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import type { BaseHttpClient, SendRequestOptions, StreamOptions } from './base-http-client.js';
|
|
2
2
|
/**
|
|
3
3
|
* A HTTP client implementation based on the `got-scraping` library.
|
|
4
4
|
*/
|
|
@@ -11,10 +11,10 @@ export declare class GotScrapingHttpClient implements BaseHttpClient {
|
|
|
11
11
|
/**
|
|
12
12
|
* @inheritDoc
|
|
13
13
|
*/
|
|
14
|
-
sendRequest
|
|
14
|
+
sendRequest(request: Request, options?: SendRequestOptions): Promise<Response>;
|
|
15
15
|
/**
|
|
16
16
|
* @inheritDoc
|
|
17
17
|
*/
|
|
18
|
-
stream(request:
|
|
18
|
+
stream(request: Request, options?: StreamOptions): Promise<Response>;
|
|
19
19
|
}
|
|
20
20
|
//# sourceMappingURL=got-scraping-http-client.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"got-scraping-http-client.d.ts","sourceRoot":"","sources":["../../src/http_clients/got-scraping-http-client.ts"],"names":[],"mappings":"AAKA,OAAO,
|
|
1
|
+
{"version":3,"file":"got-scraping-http-client.d.ts","sourceRoot":"","sources":["../../src/http_clients/got-scraping-http-client.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,cAAc,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAG/F;;GAEG;AACH,qBAAa,qBAAsB,YAAW,cAAc;IACxD;;;OAGG;IACH,OAAO,CAAC,eAAe;IAMvB;;OAEG;IACG,WAAW,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,QAAQ,CAAC;IAwCpF;;OAEG;IACG,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,QAAQ,CAAC;CAyD7E"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Readable } from 'node:stream';
|
|
2
2
|
import { gotScraping } from 'got-scraping';
|
|
3
|
-
import { ResponseWithUrl
|
|
3
|
+
import { ResponseWithUrl } from './base-http-client.js';
|
|
4
4
|
/**
|
|
5
5
|
* A HTTP client implementation based on the `got-scraping` library.
|
|
6
6
|
*/
|
|
@@ -15,19 +15,22 @@ export class GotScrapingHttpClient {
|
|
|
15
15
|
/**
|
|
16
16
|
* @inheritDoc
|
|
17
17
|
*/
|
|
18
|
-
async sendRequest(request) {
|
|
18
|
+
async sendRequest(request, options) {
|
|
19
|
+
const { session, timeout } = options ?? {};
|
|
19
20
|
if (!this.validateRequest(request)) {
|
|
20
21
|
throw new Error(`The HTTP method CONNECT is not supported by the GotScrapingHttpClient.`);
|
|
21
22
|
}
|
|
22
23
|
const gotResult = await gotScraping({
|
|
23
|
-
|
|
24
|
+
url: request.url,
|
|
25
|
+
method: request.method,
|
|
26
|
+
headers: Object.fromEntries(request.headers.entries()),
|
|
27
|
+
body: request.body ? Readable.fromWeb(request.body) : undefined,
|
|
28
|
+
proxyUrl: session?.proxyInfo?.url,
|
|
29
|
+
timeout: { request: timeout },
|
|
30
|
+
// We set cookieJar to undefined because
|
|
24
31
|
// `HttpCrawler` reads the cookies beforehand and sets them in `request.gotOptions`.
|
|
25
32
|
// Using the `cookieJar` option directly would override that.
|
|
26
33
|
cookieJar: undefined,
|
|
27
|
-
retry: {
|
|
28
|
-
limit: 0,
|
|
29
|
-
...request.retry,
|
|
30
|
-
},
|
|
31
34
|
});
|
|
32
35
|
const parsedHeaders = Object.entries(gotResult.headers)
|
|
33
36
|
.map(([key, value]) => {
|
|
@@ -49,13 +52,20 @@ export class GotScrapingHttpClient {
|
|
|
49
52
|
/**
|
|
50
53
|
* @inheritDoc
|
|
51
54
|
*/
|
|
52
|
-
async stream(request,
|
|
55
|
+
async stream(request, options) {
|
|
53
56
|
if (!this.validateRequest(request)) {
|
|
54
57
|
throw new Error(`The HTTP method CONNECT is not supported by the GotScrapingHttpClient.`);
|
|
55
58
|
}
|
|
56
59
|
// eslint-disable-next-line no-async-promise-executor
|
|
57
60
|
return new Promise(async (resolve, reject) => {
|
|
58
|
-
const stream = gotScraping({
|
|
61
|
+
const stream = gotScraping({
|
|
62
|
+
url: request.url,
|
|
63
|
+
method: request.method,
|
|
64
|
+
headers: Object.fromEntries(request.headers.entries()),
|
|
65
|
+
body: request.body ? Readable.fromWeb(request.body) : undefined,
|
|
66
|
+
isStream: true,
|
|
67
|
+
cookieJar: undefined,
|
|
68
|
+
});
|
|
59
69
|
stream.on('redirect', (updatedOptions, redirectResponse) => {
|
|
60
70
|
const nativeRedirectResponse = new ResponseWithUrl(redirectResponse.rawBody, {
|
|
61
71
|
headers: redirectResponse.headers,
|
|
@@ -63,7 +73,14 @@ export class GotScrapingHttpClient {
|
|
|
63
73
|
statusText: redirectResponse.statusMessage,
|
|
64
74
|
url: redirectResponse.url,
|
|
65
75
|
});
|
|
66
|
-
|
|
76
|
+
const nativeHeaders = new Headers(Object.entries(updatedOptions.headers)
|
|
77
|
+
.map(([key, value]) => (Array.isArray(value) ? value.map((v) => [key, v]) : [[key, value]]))
|
|
78
|
+
.flat());
|
|
79
|
+
options?.onRedirect?.(nativeRedirectResponse, {
|
|
80
|
+
url: updatedOptions.url,
|
|
81
|
+
headers: nativeHeaders,
|
|
82
|
+
});
|
|
83
|
+
updatedOptions.headers = Object.fromEntries(nativeHeaders.entries());
|
|
67
84
|
});
|
|
68
85
|
// We need to end the stream for DELETE requests, otherwise it will hang.
|
|
69
86
|
if (request.method && ['DELETE', 'delete'].includes(request.method)) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"got-scraping-http-client.js","sourceRoot":"","sources":["../../src/http_clients/got-scraping-http-client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAGvC,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"got-scraping-http-client.js","sourceRoot":"","sources":["../../src/http_clients/got-scraping-http-client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAGvC,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAG3C,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD;;GAEG;AACH,MAAM,OAAO,qBAAqB;IAC9B;;;OAGG;IACK,eAAe,CACnB,OAAgB;QAEhB,OAAO,CAAC,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAO,CAAC,CAAC;IAC7D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CAAC,OAAgB,EAAE,OAA4B;QAC5D,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,OAAO,IAAI,EAAE,CAAC;QAE3C,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CAAC,wEAAwE,CAAC,CAAC;QAC9F,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC;YAChC,GAAG,EAAE,OAAO,CAAC,GAAI;YACjB,MAAM,EAAE,OAAO,CAAC,MAA2B;YAC3C,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YACtD,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,IAAW,CAAC,CAAC,CAAC,CAAC,SAAS;YACtE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG;YACjC,OAAO,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE;YAC7B,wCAAwC;YACxC,oFAAoF;YACpF,6DAA6D;YAC7D,SAAS,EAAE,SAAS;SACvB,CAAC,CAAC;QAEH,MAAM,aAAa,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC;aAClD,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;YAClB,IAAI,KAAK,KAAK,SAAS;gBAAE,OAAO,EAAE,CAAC;YAEnC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;gBACvB,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;YACtC,CAAC;YAED,OAAO,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;QAC1B,CAAC,CAAC;aACD,IAAI,EAAwB,CAAC;QAElC,OAAO,IAAI,eAAe,CAAC,IAAI,UAAU,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE;YAC1D,OAAO,EAAE,IAAI,OAAO,CAAC,aAAa,CAAC;YACnC,MAAM,EAAE,SAAS,CAAC,UAAU;YAC5B,UAAU,EAAE,SAAS,CAAC,aAAa,IAAI,EAAE;YACzC,GAAG,EAAE,SAAS,CAAC,GAAG;SACrB,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,OAAgB,EAAE,OAAuB;QAClD,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CAAC,wEAAwE,CAAC,CAAC;QAC9F,CAAC;QACD,qDAAqD;QACrD,OAAO,IAAI,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE;YACzC,MAAM,MAAM,GAAG,WAAW,CAAC;gBACvB,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,MAAM,EAAE,OAAO,CAAC,MAA2B;gBAC3C,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;gBACtD,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,IAAW,CAAC,CAAC,CAAC,CAAC,SAAS;gBACtE,QAAQ,EAAE,IAAI;gBACd,SAAS,EAAE,SAAS;aACvB,CAAC,CAAC;YAEH,MAAM,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,cAAuB,EAAE,gBAAqB,EAAE,EAAE;gBACrE,MAAM,sBAAsB,GAAG,IAAI,eAAe,CAAC,gBAAgB,CAAC,OAAO,EAAE;oBACzE,OAAO,EAAE,gBAAgB,CAAC,OAAO;oBACjC,MAAM,EAAE,gBAAgB,CAAC,UAAU;oBACnC,UAAU,EAAE,gBAAgB,CAAC,aAAa;oBAC1C,GAAG,EAAE,gBAAgB,CAAC,GAAG;iBAC5B,CAAC,CAAC;gBAEH,MAAM,aAAa,GAAG,IAAI,OAAO,CAC7B,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,OAAO,CAAC;qBACjC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;qBAC3F,IAAI,EAAwB,CACpC,CAAC;gBAEF,OAAO,EAAE,UAAU,EAAE,CAAC,sBAAsB,EAAE;oBAC1C,GAAG,EAAE,cAAc,CAAC,GAAG;oBACvB,OAAO,EAAE,aAAa;iBACzB,CAAC,CAAC;gBAEH,cAAc,CAAC,OAAO,GAAG,MAAM,CAAC,WAAW,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC;YACzE,CAAC,CAAC,CAAC;YAEH,yEAAyE;YACzE,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAClE,MAAM,CAAC,GAAG,EAAE,CAAC;YACjB,CAAC;YAED,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAE3B,MAAM,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,QAAuB,EAAE,EAAE;gBAC9C,qFAAqF;gBACrF,OAAO,CACH,IAAI,eAAe,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAQ,EAAE;oBAC/C,MAAM,EAAE,QAAQ,CAAC,UAAU;oBAC3B,UAAU,EAAE,QAAQ,CAAC,aAAa,IAAI,EAAE;oBACxC,OAAO,EAAE,QAAQ,CAAC,OAAsB;oBACxC,GAAG,EAAE,QAAQ,CAAC,GAAG;iBACpB,CAAC,CACL,CAAC;YACN,CAAC,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;IACP,CAAC;CACJ"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/core",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.18",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -53,9 +53,9 @@
|
|
|
53
53
|
"@apify/pseudo_url": "^2.0.59",
|
|
54
54
|
"@apify/timeout": "^0.3.2",
|
|
55
55
|
"@apify/utilities": "^2.15.5",
|
|
56
|
-
"@crawlee/memory-storage": "4.0.0-beta.
|
|
57
|
-
"@crawlee/types": "4.0.0-beta.
|
|
58
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
56
|
+
"@crawlee/memory-storage": "4.0.0-beta.18",
|
|
57
|
+
"@crawlee/types": "4.0.0-beta.18",
|
|
58
|
+
"@crawlee/utils": "4.0.0-beta.18",
|
|
59
59
|
"@sapphire/async-queue": "^1.5.5",
|
|
60
60
|
"@vladfrangu/async_event_emitter": "^2.4.6",
|
|
61
61
|
"csv-stringify": "^6.5.2",
|
|
@@ -77,5 +77,5 @@
|
|
|
77
77
|
}
|
|
78
78
|
}
|
|
79
79
|
},
|
|
80
|
-
"gitHead": "
|
|
80
|
+
"gitHead": "cfaba5f4e082914d98d96d6539ab7b324d1c14c7"
|
|
81
81
|
}
|