@crawlee/impit-client 4.0.0-rc.0 → 4.0.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/index.d.ts +3 -5
- package/index.js +4 -4
- package/package.json +6 -6
package/README.md
CHANGED
|
@@ -34,7 +34,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
|
|
|
34
34
|
|
|
35
35
|
We recommend visiting the [Introduction tutorial](https://crawlee.dev/js/docs/introduction) in Crawlee documentation for more information.
|
|
36
36
|
|
|
37
|
-
> Crawlee requires **Node.js
|
|
37
|
+
> Crawlee requires **Node.js 22.13 or higher**.
|
|
38
38
|
|
|
39
39
|
### With Crawlee CLI
|
|
40
40
|
|
package/index.d.ts
CHANGED
|
@@ -3,15 +3,14 @@ import { BaseHttpClient } from '@crawlee/http-client';
|
|
|
3
3
|
import type { CrawleeLogger } from '@crawlee/types';
|
|
4
4
|
import { type ImpitOptions } from 'impit';
|
|
5
5
|
export declare const Browser: {
|
|
6
|
-
readonly Chrome:
|
|
7
|
-
readonly Firefox:
|
|
6
|
+
readonly Chrome: 'chrome';
|
|
7
|
+
readonly Firefox: 'firefox';
|
|
8
8
|
};
|
|
9
9
|
/**
|
|
10
10
|
* A HTTP client implementation based on the `impit` library.
|
|
11
11
|
*/
|
|
12
12
|
export declare class ImpitHttpClient extends BaseHttpClient {
|
|
13
13
|
#private;
|
|
14
|
-
private getClient;
|
|
15
14
|
/**
|
|
16
15
|
* @param options.cacheClients Whether to cache `impit` clients between requests. Defaults to `true`.
|
|
17
16
|
*/
|
|
@@ -22,6 +21,5 @@ export declare class ImpitHttpClient extends BaseHttpClient {
|
|
|
22
21
|
/**
|
|
23
22
|
* @inheritDoc
|
|
24
23
|
*/
|
|
25
|
-
fetch(request: Request, options?: RequestInit & CustomFetchOptions): Promise<Response>;
|
|
26
|
-
private resolveImpitBrowser;
|
|
24
|
+
protected fetch(request: Request, options?: RequestInit & CustomFetchOptions): Promise<Response>;
|
|
27
25
|
}
|
package/index.js
CHANGED
|
@@ -44,7 +44,7 @@ export class ImpitHttpClient extends BaseHttpClient {
|
|
|
44
44
|
* instead of rerolling on every call.
|
|
45
45
|
*/
|
|
46
46
|
#impitBrowserByFingerprint = new WeakMap();
|
|
47
|
-
getClient(options) {
|
|
47
|
+
#getClient(options) {
|
|
48
48
|
if (!this.#cacheClients) {
|
|
49
49
|
return new Impit(options);
|
|
50
50
|
}
|
|
@@ -72,8 +72,8 @@ export class ImpitHttpClient extends BaseHttpClient {
|
|
|
72
72
|
*/
|
|
73
73
|
async fetch(request, options) {
|
|
74
74
|
const { proxyUrl, redirect, signal, fingerprint, ignoreTlsErrors } = options ?? {};
|
|
75
|
-
const impitBrowser = this
|
|
76
|
-
const impit = this
|
|
75
|
+
const impitBrowser = this.#resolveImpitBrowser(fingerprint);
|
|
76
|
+
const impit = this.#getClient({
|
|
77
77
|
...this.#impitOptions,
|
|
78
78
|
...(impitBrowser ? { browser: impitBrowser } : {}),
|
|
79
79
|
// The per-request flag (from the crawler option or a MITM proxy session)
|
|
@@ -86,7 +86,7 @@ export class ImpitHttpClient extends BaseHttpClient {
|
|
|
86
86
|
// todo - cast shouldn't be needed here, impit returns `Uint8Array`
|
|
87
87
|
return new ResponseWithUrl(response.body, response);
|
|
88
88
|
}
|
|
89
|
-
resolveImpitBrowser(fingerprint) {
|
|
89
|
+
#resolveImpitBrowser(fingerprint) {
|
|
90
90
|
if (!fingerprint?.browser)
|
|
91
91
|
return undefined;
|
|
92
92
|
const cached = this.#impitBrowserByFingerprint.get(fingerprint);
|
package/package.json
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/impit-client",
|
|
3
|
-
"version": "4.0.0-rc.
|
|
3
|
+
"version": "4.0.0-rc.1",
|
|
4
4
|
"description": "impit-based HTTP client implementation for Crawlee. Impersonates browser requests to avoid bot detection.",
|
|
5
5
|
"engines": {
|
|
6
|
-
"node": ">=22.
|
|
6
|
+
"node": ">=22.13.0"
|
|
7
7
|
},
|
|
8
8
|
"type": "module",
|
|
9
9
|
"exports": {
|
|
@@ -47,9 +47,9 @@
|
|
|
47
47
|
"access": "public"
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
|
-
"@apify/datastructures": "^
|
|
51
|
-
"@crawlee/http-client": "4.0.0-rc.
|
|
52
|
-
"@crawlee/types": "4.0.0-rc.
|
|
50
|
+
"@apify/datastructures": "^3.0.1",
|
|
51
|
+
"@crawlee/http-client": "4.0.0-rc.1",
|
|
52
|
+
"@crawlee/types": "4.0.0-rc.1",
|
|
53
53
|
"impit": "^0.14.2",
|
|
54
54
|
"tough-cookie": "^6.0.0"
|
|
55
55
|
},
|
|
@@ -60,5 +60,5 @@
|
|
|
60
60
|
}
|
|
61
61
|
}
|
|
62
62
|
},
|
|
63
|
-
"gitHead": "
|
|
63
|
+
"gitHead": "f354ca5e943bed1a5c657a1fce1974c053f9fff0"
|
|
64
64
|
}
|