@crawlee/impit-client 4.0.0-beta.17 → 4.0.0-beta.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +3 -10
- package/index.d.ts.map +1 -1
- package/index.js +8 -44
- package/index.js.map +1 -1
- package/package.json +2 -2
package/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import type { BaseHttpClient, SendRequestOptions, StreamOptions } from '@crawlee/core';
|
|
2
2
|
import type { ImpitOptions } from 'impit';
|
|
3
3
|
export declare const Browser: {
|
|
4
4
|
readonly Chrome: "chrome";
|
|
@@ -22,13 +22,6 @@ export declare class ImpitHttpClient implements BaseHttpClient {
|
|
|
22
22
|
constructor(options?: Omit<ImpitOptions, 'proxyUrl'> & {
|
|
23
23
|
maxRedirects?: number;
|
|
24
24
|
});
|
|
25
|
-
/**
|
|
26
|
-
* Flattens the headers of a `HttpRequest` to a format that can be passed to `impit`.
|
|
27
|
-
* @param headers `SimpleHeaders` object
|
|
28
|
-
* @returns `Record<string, string>` object
|
|
29
|
-
*/
|
|
30
|
-
private intoHeaders;
|
|
31
|
-
private intoImpitBody;
|
|
32
25
|
/**
|
|
33
26
|
* Common implementation for `sendRequest` and `stream` methods.
|
|
34
27
|
* @param request `HttpRequest` object
|
|
@@ -38,11 +31,11 @@ export declare class ImpitHttpClient implements BaseHttpClient {
|
|
|
38
31
|
/**
|
|
39
32
|
* @inheritDoc
|
|
40
33
|
*/
|
|
41
|
-
sendRequest
|
|
34
|
+
sendRequest(request: Request, options?: SendRequestOptions): Promise<Response>;
|
|
42
35
|
private getStreamWithProgress;
|
|
43
36
|
/**
|
|
44
37
|
* @inheritDoc
|
|
45
38
|
*/
|
|
46
|
-
stream(request:
|
|
39
|
+
stream(request: Request, options?: StreamOptions): Promise<Response>;
|
|
47
40
|
}
|
|
48
41
|
//# sourceMappingURL=index.d.ts.map
|
package/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAEvF,OAAO,KAAK,EAAE,YAAY,EAAiB,MAAM,OAAO,CAAC;AAMzD,eAAO,MAAM,OAAO;;;CAGV,CAAC;AAOX;;GAEG;AACH,qBAAa,eAAgB,YAAW,cAAc;IAClD,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,eAAe,CAAU;IAEjC;;;;;OAKG;IACH,OAAO,CAAC,WAAW,CAA2F;IAE9G,OAAO,CAAC,SAAS;gBAgBL,OAAO,CAAC,EAAE,IAAI,CAAC,YAAY,EAAE,UAAU,CAAC,GAAG;QAAE,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE;IAOhF;;;;OAIG;YACW,WAAW;IA+CzB;;OAEG;IACG,WAAW,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,QAAQ,CAAC;IAOpF,OAAO,CAAC,qBAAqB;IAqB7B;;OAEG;IACG,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,QAAQ,CAAC;CAO7E"}
|
package/index.js
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { Readable } from 'node:stream';
|
|
2
|
-
import { isGeneratorObject } from 'node:util/types';
|
|
3
2
|
import { ResponseWithUrl } from '@crawlee/core';
|
|
4
3
|
import { Impit } from 'impit';
|
|
5
4
|
import { LruCache } from '@apify/datastructures';
|
|
@@ -37,57 +36,22 @@ export class ImpitHttpClient {
|
|
|
37
36
|
this.maxRedirects = options?.maxRedirects ?? 10;
|
|
38
37
|
this.followRedirects = options?.followRedirects ?? true;
|
|
39
38
|
}
|
|
40
|
-
/**
|
|
41
|
-
* Flattens the headers of a `HttpRequest` to a format that can be passed to `impit`.
|
|
42
|
-
* @param headers `SimpleHeaders` object
|
|
43
|
-
* @returns `Record<string, string>` object
|
|
44
|
-
*/
|
|
45
|
-
intoHeaders(headers) {
|
|
46
|
-
if (!headers) {
|
|
47
|
-
return undefined;
|
|
48
|
-
}
|
|
49
|
-
const result = new Headers();
|
|
50
|
-
for (const headerName of Object.keys(headers)) {
|
|
51
|
-
const headerValue = headers[headerName];
|
|
52
|
-
for (const value of Array.isArray(headerValue) ? headerValue : [headerValue]) {
|
|
53
|
-
if (value === undefined)
|
|
54
|
-
continue;
|
|
55
|
-
result.append(headerName, value);
|
|
56
|
-
}
|
|
57
|
-
}
|
|
58
|
-
return result;
|
|
59
|
-
}
|
|
60
|
-
intoImpitBody(body) {
|
|
61
|
-
if (isGeneratorObject(body)) {
|
|
62
|
-
return Readable.toWeb(Readable.from(body));
|
|
63
|
-
}
|
|
64
|
-
if (body instanceof Readable) {
|
|
65
|
-
return Readable.toWeb(body);
|
|
66
|
-
}
|
|
67
|
-
return body;
|
|
68
|
-
}
|
|
69
39
|
/**
|
|
70
40
|
* Common implementation for `sendRequest` and `stream` methods.
|
|
71
41
|
* @param request `HttpRequest` object
|
|
72
42
|
* @returns `HttpResponse` object
|
|
73
43
|
*/
|
|
74
|
-
async getResponse(request, redirects) {
|
|
44
|
+
async getResponse(request, redirects, options) {
|
|
75
45
|
if ((redirects?.redirectCount ?? 0) > this.maxRedirects) {
|
|
76
46
|
throw new Error(`Too many redirects, maximum is ${this.maxRedirects}.`);
|
|
77
47
|
}
|
|
78
|
-
const url = typeof request.url === 'string' ? request.url : request.url.href;
|
|
79
48
|
const impit = this.getClient({
|
|
80
49
|
...this.impitOptions,
|
|
81
|
-
...(
|
|
82
|
-
proxyUrl:
|
|
50
|
+
...(options?.cookieJar ? { cookieJar: options?.cookieJar } : {}),
|
|
51
|
+
proxyUrl: options?.session?.proxyInfo?.url,
|
|
83
52
|
followRedirects: false,
|
|
84
53
|
});
|
|
85
|
-
const response = await impit.fetch(
|
|
86
|
-
method: request.method,
|
|
87
|
-
headers: this.intoHeaders(request.headers),
|
|
88
|
-
body: this.intoImpitBody(request.body),
|
|
89
|
-
timeout: request.timeout?.request,
|
|
90
|
-
});
|
|
54
|
+
const response = await impit.fetch(request, { timeout: options?.timeout });
|
|
91
55
|
if (this.followRedirects && response.status >= 300 && response.status < 400) {
|
|
92
56
|
const location = response.headers.get('location');
|
|
93
57
|
const redirectUrl = new URL(location ?? '', request.url);
|
|
@@ -110,8 +74,8 @@ export class ImpitHttpClient {
|
|
|
110
74
|
/**
|
|
111
75
|
* @inheritDoc
|
|
112
76
|
*/
|
|
113
|
-
async sendRequest(request) {
|
|
114
|
-
const { response } = await this.getResponse(request);
|
|
77
|
+
async sendRequest(request, options) {
|
|
78
|
+
const { response } = await this.getResponse(request, {}, options);
|
|
115
79
|
// todo - cast shouldn't be needed here, impit returns `Uint8Array`
|
|
116
80
|
return new ResponseWithUrl((await response.bytes()), response);
|
|
117
81
|
}
|
|
@@ -134,8 +98,8 @@ export class ImpitHttpClient {
|
|
|
134
98
|
/**
|
|
135
99
|
* @inheritDoc
|
|
136
100
|
*/
|
|
137
|
-
async stream(request) {
|
|
138
|
-
const { response } = await this.getResponse(request);
|
|
101
|
+
async stream(request, options) {
|
|
102
|
+
const { response } = await this.getResponse(request, {}, options);
|
|
139
103
|
const [stream] = this.getStreamWithProgress(response);
|
|
140
104
|
// Cast shouldn't be needed here, undici might have a slightly different `ReadableStream` type
|
|
141
105
|
return new ResponseWithUrl(Readable.toWeb(stream), response);
|
package/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAIvC,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAEhD,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAG9B,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAEjD,MAAM,CAAC,MAAM,OAAO,GAAG;IACnB,QAAQ,EAAE,QAAQ;IAClB,SAAS,EAAE,SAAS;CACd,CAAC;AAOX;;GAEG;AACH,MAAM,OAAO,eAAe;IAChB,YAAY,CAAe;IAC3B,YAAY,CAAS;IACrB,eAAe,CAAU;IAEjC;;;;;OAKG;IACK,WAAW,GAA2D,IAAI,QAAQ,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC;IAEtG,SAAS,CAAC,OAAqB;QACnC,MAAM,EAAE,SAAS,EAAE,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC;QAEvC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,cAAc,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEtD,IAAI,cAAc,IAAI,CAAC,CAAC,SAAS,IAAI,cAAc,CAAC,SAAS,KAAK,SAAS,CAAC,EAAE,CAAC;YAC3E,OAAO,cAAc,CAAC,MAAM,CAAC;QACjC,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC;QAClC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,SAA2B,EAAE,CAAC,CAAC;QAEnF,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,YAAY,OAAoE;QAC5E,IAAI,CAAC,YAAY,GAAG,OAAO,IAAI,EAAE,CAAC;QAElC,IAAI,CAAC,YAAY,GAAG,OAAO,EAAE,YAAY,IAAI,EAAE,CAAC;QAChD,IAAI,CAAC,eAAe,GAAG,OAAO,EAAE,eAAe,IAAI,IAAI,CAAC;IAC5D,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,WAAW,CACrB,OAAgB,EAChB,SAGC,EACD,OAAuB;QAEvB,IAAI,CAAC,SAAS,EAAE,aAAa,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;YACtD,MAAM,IAAI,KAAK,CAAC,kCAAkC,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;QAC5E,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC;YACzB,GAAG,IAAI,CAAC,YAAY;YACpB,GAAG,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,SAAkC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzF,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG;YAC1C,eAAe,EAAE,KAAK;SACzB,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;QAE3E,IAAI,IAAI,CAAC,eAAe,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YAC1E,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAClD,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,QAAQ,IAAI,EAAE,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;YAEzD,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;YAClE,CAAC;YAED,OAAO,IAAI,CAAC,WAAW,CACnB;gBACI,GAAG,OAAO;gBACV,GAAG,EAAE,WAAW,CAAC,IAAI;aACxB,EACD;gBACI,aAAa,EAAE,CAAC,SAAS,EAAE,aAAa,IAAI,CAAC,CAAC,GAAG,CAAC;gBAClD,YAAY,EAAE,CAAC,GAAG,CAAC,SAAS,EAAE,YAAY,IAAI,EAAE,CAAC,EAAE,WAAW,CAAC;aAClE,CACJ,CAAC;QACN,CAAC;QAED,OAAO;YACH,QAAQ;YACR,YAAY,EAAE,SAAS,EAAE,YAAY,IAAI,EAAE;SAC9C,CAAC;IACN,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CAAC,OAAgB,EAAE,OAA4B;QAC5D,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC;QAElE,mEAAmE;QACnE,OAAO,IAAI,eAAe,CAAC,CAAC,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAQ,EAAE,QAAQ,CAAC,CAAC;IAC1E,CAAC;IAEO,qBAAqB,CACzB,QAAuB;QAEvB,MAAM,cAAc,GAAG,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,IAA2B,CAAC,CAAC;QAC9E,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC;QAClE,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;YAChC,WAAW,IAAI,KAAK,CAAC,MAAM,CAAC;QAChC,CAAC,CAAC,CAAC;QAEH,MAAM,mBAAmB,GAAG,GAAG,EAAE;YAC7B,OAAO;gBACH,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,WAAW,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC;gBAChD,WAAW;gBACX,KAAK;aACR,CAAC;QACN,CAAC,CAAC;QAEF,OAAO,CAAC,cAAc,EAAE,mBAAmB,CAAC,CAAC;IACjD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,OAAgB,EAAE,OAAuB;QAClD,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC;QAClE,MAAM,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAC;QAEtD,8FAA8F;QAC9F,OAAO,IAAI,eAAe,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAQ,EAAE,QAAQ,CAAC,CAAC;IACxE,CAAC;CACJ"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/impit-client",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.19",
|
|
4
4
|
"description": "impit-based HTTP client implementation for Crawlee. Impersonates browser requests to avoid bot detection.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -64,5 +64,5 @@
|
|
|
64
64
|
}
|
|
65
65
|
}
|
|
66
66
|
},
|
|
67
|
-
"gitHead": "
|
|
67
|
+
"gitHead": "690af03140bea34c9a8871d370607cd52339caa0"
|
|
68
68
|
}
|