@crawlee/impit-client 3.13.5 → 4.0.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.js +22 -46
- package/index.js.map +1 -1
- package/package.json +9 -16
- package/tsconfig.build.tsbuildinfo +1 -1
- package/index.mjs +0 -5
package/index.js
CHANGED
|
@@ -1,58 +1,35 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
const
|
|
6
|
-
const impit_1 = require("impit");
|
|
7
|
-
const datastructures_1 = require("@apify/datastructures");
|
|
8
|
-
exports.Browser = {
|
|
1
|
+
import { Readable } from 'node:stream';
|
|
2
|
+
import { isGeneratorObject } from 'node:util/types';
|
|
3
|
+
import { Impit } from 'impit';
|
|
4
|
+
import { LruCache } from '@apify/datastructures';
|
|
5
|
+
export const Browser = {
|
|
9
6
|
'Chrome': 'chrome',
|
|
10
7
|
'Firefox': 'firefox',
|
|
11
8
|
};
|
|
12
9
|
/**
|
|
13
10
|
* A HTTP client implementation based on the `impit library.
|
|
14
11
|
*/
|
|
15
|
-
class ImpitHttpClient {
|
|
12
|
+
export class ImpitHttpClient {
|
|
13
|
+
impitOptions;
|
|
14
|
+
maxRedirects;
|
|
15
|
+
followRedirects;
|
|
16
|
+
/**
|
|
17
|
+
* Enables reuse of `impit` clients for the same set of options.
|
|
18
|
+
* This is useful for performance reasons, as creating
|
|
19
|
+
* a new client for each request breaks TCP connection
|
|
20
|
+
* (and other resources) reuse.
|
|
21
|
+
*/
|
|
22
|
+
clientCache = new LruCache({ maxLength: 10 });
|
|
16
23
|
getClient(options) {
|
|
17
24
|
const cacheKey = JSON.stringify(options);
|
|
18
25
|
if (this.clientCache.get(cacheKey)) {
|
|
19
26
|
return this.clientCache.get(cacheKey);
|
|
20
27
|
}
|
|
21
|
-
const client = new
|
|
28
|
+
const client = new Impit(options);
|
|
22
29
|
this.clientCache.add(cacheKey, client);
|
|
23
30
|
return client;
|
|
24
31
|
}
|
|
25
32
|
constructor(options) {
|
|
26
|
-
Object.defineProperty(this, "impitOptions", {
|
|
27
|
-
enumerable: true,
|
|
28
|
-
configurable: true,
|
|
29
|
-
writable: true,
|
|
30
|
-
value: void 0
|
|
31
|
-
});
|
|
32
|
-
Object.defineProperty(this, "maxRedirects", {
|
|
33
|
-
enumerable: true,
|
|
34
|
-
configurable: true,
|
|
35
|
-
writable: true,
|
|
36
|
-
value: void 0
|
|
37
|
-
});
|
|
38
|
-
Object.defineProperty(this, "followRedirects", {
|
|
39
|
-
enumerable: true,
|
|
40
|
-
configurable: true,
|
|
41
|
-
writable: true,
|
|
42
|
-
value: void 0
|
|
43
|
-
});
|
|
44
|
-
/**
|
|
45
|
-
* Enables reuse of `impit` clients for the same set of options.
|
|
46
|
-
* This is useful for performance reasons, as creating
|
|
47
|
-
* a new client for each request breaks TCP connection
|
|
48
|
-
* (and other resources) reuse.
|
|
49
|
-
*/
|
|
50
|
-
Object.defineProperty(this, "clientCache", {
|
|
51
|
-
enumerable: true,
|
|
52
|
-
configurable: true,
|
|
53
|
-
writable: true,
|
|
54
|
-
value: new datastructures_1.LruCache({ maxLength: 10 })
|
|
55
|
-
});
|
|
56
33
|
this.impitOptions = options ?? {};
|
|
57
34
|
this.maxRedirects = options?.maxRedirects ?? 10;
|
|
58
35
|
this.followRedirects = options?.followRedirects ?? true;
|
|
@@ -78,11 +55,11 @@ class ImpitHttpClient {
|
|
|
78
55
|
return result;
|
|
79
56
|
}
|
|
80
57
|
intoImpitBody(body) {
|
|
81
|
-
if (
|
|
82
|
-
return
|
|
58
|
+
if (isGeneratorObject(body)) {
|
|
59
|
+
return Readable.toWeb(Readable.from(body));
|
|
83
60
|
}
|
|
84
|
-
if (body instanceof
|
|
85
|
-
return
|
|
61
|
+
if (body instanceof Readable) {
|
|
62
|
+
return Readable.toWeb(body);
|
|
86
63
|
}
|
|
87
64
|
return body;
|
|
88
65
|
}
|
|
@@ -156,7 +133,7 @@ class ImpitHttpClient {
|
|
|
156
133
|
};
|
|
157
134
|
}
|
|
158
135
|
getStreamWithProgress(response) {
|
|
159
|
-
const responseStream =
|
|
136
|
+
const responseStream = Readable.fromWeb(response.body);
|
|
160
137
|
let transferred = 0;
|
|
161
138
|
const total = Number(response.headers.get('content-length') ?? 0);
|
|
162
139
|
responseStream.on('data', (chunk) => {
|
|
@@ -193,5 +170,4 @@ class ImpitHttpClient {
|
|
|
193
170
|
};
|
|
194
171
|
}
|
|
195
172
|
}
|
|
196
|
-
exports.ImpitHttpClient = ImpitHttpClient;
|
|
197
173
|
//# sourceMappingURL=index.js.map
|
package/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEvC,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAIpD,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAE9B,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAEjD,MAAM,CAAC,MAAM,OAAO,GAAG;IACnB,QAAQ,EAAE,QAAQ;IAClB,SAAS,EAAE,SAAS;CACd,CAAC;AAOX;;GAEG;AACH,MAAM,OAAO,eAAe;IAChB,YAAY,CAAe;IAC3B,YAAY,CAAS;IACrB,eAAe,CAAU;IAEjC;;;;;OAKG;IACK,WAAW,GAAoB,IAAI,QAAQ,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC;IAE/D,SAAS,CAAC,OAAqB;QACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAEzC,IAAI,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YACjC,OAAO,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC;QAC3C,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC;QAClC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAEvC,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,YAAY,OAAoE;QAC5E,IAAI,CAAC,YAAY,GAAG,OAAO,IAAI,EAAE,CAAC;QAElC,IAAI,CAAC,YAAY,GAAG,OAAO,EAAE,YAAY,IAAI,EAAE,CAAC;QAChD,IAAI,CAAC,eAAe,GAAG,OAAO,EAAE,eAAe,IAAI,IAAI,CAAC;IAC5D,CAAC;IAED;;;;OAIG;IACK,WAAW,CACf,OAAmE;QAEnE,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,OAAO,SAAS,CAAC;QACrB,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAE7B,KAAK,MAAM,UAAU,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC5C,MAAM,WAAW,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;YAExC,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC3E,IAAI,KAAK,KAAK,SAAS;oBAAE,SAAS;gBAElC,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;YACrC,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAEO,aAAa,CACjB,IAA6D;QAE7D,IAAI,iBAAiB,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1B,OAAO,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAQ,CAAC;QACtD,CAAC;QACD,IAAI,IAAI,YAAY,QAAQ,EAAE,CAAC;YAC3B,OAAO,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAQ,CAAC;QACvC,CAAC;QAED,OAAO,IAAW,CAAC;IACvB,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,WAAW,CACrB,OAAmC,EACnC,SAGC;QAED,IAAI,CAAC,SAAS,EAAE,aAAa,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;YACtD,MAAM,IAAI,KAAK,CAAC,kCAAkC,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;QAC5E,CAAC;QAED,MAAM,GAAG,GAAG,OAAO,OAAO,CAAC,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC;QAE7E,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC;YACzB,GAAG,IAAI,CAAC,YAAY;YACpB,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,eAAe,EAAE,KAAK;SACzB,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE;YACpC,MAAM,EAAE,OAAO,CAAC,MAAoB;YACpC,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC;YAC1C,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,IAAI,CAAC;SACzC,CAAC,CAAC;QAEH,IAAI,IAAI,CAAC,eAAe,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YAC1E,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAClD,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,QAAQ,IAAI,EAAE,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;YAEzD,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;YAClE,CAAC;YAED,OAAO,IAAI,CAAC,WAAW,CACnB;gBACI,GAAG,OAAO;gBACV,GAAG,EAAE,WAAW,CAAC,IAAI;aACxB,EACD;gBACI,aAAa,EAAE,CAAC,SAAS,EAAE,aAAa,IAAI,CAAC,CAAC,GAAG,CAAC;gBAClD,YAAY,EAAE,CAAC,GAAG,CAAC,SAAS,EAAE,YAAY,IAAI,EAAE,CAAC,EAAE,WAAW,CAAC;aAClE,CACJ,CAAC;QACN,CAAC;QAED,OAAO;YACH,QAAQ;YACR,YAAY,EAAE,SAAS,EAAE,YAAY,IAAI,EAAE;SAC9C,CAAC;IACN,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CACb,OAAmC;QAEnC,MAAM,EAAE,QAAQ,EAAE,YAAY,EAAE,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;QAEnE,IAAI,YAAY,CAAC;QAEjB,QAAQ,OAAO,CAAC,YAAY,EAAE,CAAC;YAC3B,KAAK,MAAM;gBACP,YAAY,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACrC,MAAM;YACV,KAAK,MAAM;gBACP,YAAY,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACrC,MAAM;YACV,KAAK,QAAQ;gBACT,YAAY,GAAG,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAC;gBACtC,MAAM;YACV;gBACI,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;QACtD,CAAC;QAED,OAAO;YACH,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YACvD,UAAU,EAAE,QAAQ,CAAC,MAAM;YAC3B,GAAG,EAAE,QAAQ,CAAC,GAAG;YACjB,OAAO;YACP,YAAY;YACZ,QAAQ,EAAE,EAAE;YACZ,IAAI,EAAE,YAAY;YAClB,QAAQ,EAAE,IAAI;SACjB,CAAC;IACN,CAAC;IAEO,qBAAqB,CACzB,QAAuB;QAEvB,MAAM,cAAc,GAAG,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,IAA2B,CAAC,CAAC;QAC9E,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC;QAClE,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;YAChC,WAAW,IAAI,KAAK,CAAC,MAAM,CAAC;QAChC,CAAC,CAAC,CAAC;QAEH,MAAM,mBAAmB,GAAG,GAAG,EAAE;YAC7B,OAAO;gBACH,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,WAAW,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC;gBAChD,WAAW;gBACX,KAAK;aACR,CAAC;QACN,CAAC,CAAC;QAEF,OAAO,CAAC,cAAc,EAAE,mBAAmB,CAAC,CAAC;IACjD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,OAAoB;QAC7B,MAAM,EAAE,QAAQ,EAAE,YAAY,EAAE,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;QACnE,MAAM,CAAC,MAAM,EAAE,mBAAmB,CAAC,GAAG,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAC;QAE3E,OAAO;YACH,OAAO;YACP,GAAG,EAAE,QAAQ,CAAC,GAAG;YACjB,UAAU,EAAE,QAAQ,CAAC,MAAM;YAC3B,MAAM;YACN,QAAQ,EAAE,IAAI;YACd,IAAI,gBAAgB;gBAChB,OAAO,mBAAmB,EAAE,CAAC;YACjC,CAAC;YACD,cAAc,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,CAAC,EAAE;YAChD,YAAY;YACZ,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YACvD,QAAQ,EAAE,EAAE;SACf,CAAC;IACN,CAAC;CACJ"}
|
package/package.json
CHANGED
|
@@ -1,19 +1,13 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/impit-client",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "4.0.0-beta.0",
|
|
4
4
|
"description": "impit-based HTTP client implementation for Crawlee. Impersonates browser requests to avoid bot detection.",
|
|
5
5
|
"engines": {
|
|
6
|
-
"node": ">=
|
|
6
|
+
"node": ">=22.0.0"
|
|
7
7
|
},
|
|
8
|
-
"
|
|
9
|
-
"module": "./index.mjs",
|
|
10
|
-
"types": "./index.d.ts",
|
|
8
|
+
"type": "module",
|
|
11
9
|
"exports": {
|
|
12
|
-
".":
|
|
13
|
-
"import": "./index.mjs",
|
|
14
|
-
"require": "./index.js",
|
|
15
|
-
"types": "./index.d.ts"
|
|
16
|
-
},
|
|
10
|
+
".": "./index.js",
|
|
17
11
|
"./package.json": "./package.json"
|
|
18
12
|
},
|
|
19
13
|
"keywords": [
|
|
@@ -46,23 +40,22 @@
|
|
|
46
40
|
"scripts": {
|
|
47
41
|
"build": "yarn clean && yarn compile && yarn copy",
|
|
48
42
|
"clean": "rimraf ./dist",
|
|
49
|
-
"compile": "tsc -p tsconfig.build.json
|
|
43
|
+
"compile": "tsc -p tsconfig.build.json",
|
|
50
44
|
"copy": "tsx ../../scripts/copy.ts"
|
|
51
45
|
},
|
|
52
46
|
"publishConfig": {
|
|
53
47
|
"access": "public"
|
|
54
48
|
},
|
|
55
49
|
"peerDependencies": {
|
|
56
|
-
"@crawlee/core": "^3.
|
|
50
|
+
"@crawlee/core": "^3.13.3"
|
|
57
51
|
},
|
|
58
52
|
"devDependencies": {
|
|
59
|
-
"@crawlee/core": "^3.13.
|
|
53
|
+
"@crawlee/core": "^3.13.3"
|
|
60
54
|
},
|
|
61
55
|
"dependencies": {
|
|
62
56
|
"@apify/datastructures": "^2.0.3",
|
|
63
|
-
"impit": "^0.4.
|
|
57
|
+
"impit": "^0.4.5"
|
|
64
58
|
},
|
|
65
|
-
"packageManager": "yarn@4.8.1",
|
|
66
59
|
"lerna": {
|
|
67
60
|
"command": {
|
|
68
61
|
"publish": {
|
|
@@ -70,5 +63,5 @@
|
|
|
70
63
|
}
|
|
71
64
|
}
|
|
72
65
|
},
|
|
73
|
-
"gitHead": "
|
|
66
|
+
"gitHead": "927bdafa403ff347327158b01d20b817378168a7"
|
|
74
67
|
}
|