@crawlee/impit-client 4.0.0-beta.239 → 4.0.0-beta.240
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.js +10 -2
- package/package.json +6 -6
package/index.js
CHANGED
|
@@ -87,9 +87,17 @@ export class ImpitHttpClient extends BaseHttpClient {
|
|
|
87
87
|
proxyUrl,
|
|
88
88
|
followRedirects: redirect === 'follow',
|
|
89
89
|
});
|
|
90
|
-
|
|
90
|
+
// The caller's signal carries the deadline (for crawlers, the navigation window that `extendTimeout()` can
|
|
91
|
+
// push out), so impit's 30 s default must not cut the request short. Without a signal, the default stays.
|
|
92
|
+
const response = await impit.fetch(request, {
|
|
93
|
+
signal: signal ?? undefined,
|
|
94
|
+
...(signal ? { timeout: null } : {}),
|
|
95
|
+
});
|
|
96
|
+
// Once the headers are in, an aborted signal only stops impit's body reads, while the download itself keeps
|
|
97
|
+
// going. Piping the body through the signal cancels the download too.
|
|
98
|
+
const body = signal && response.body ? response.body.pipeThrough(new TransformStream(), { signal }) : response.body;
|
|
91
99
|
// todo - cast shouldn't be needed here, impit returns `Uint8Array`
|
|
92
|
-
return new ResponseWithUrl(
|
|
100
|
+
return new ResponseWithUrl(body, response);
|
|
93
101
|
}
|
|
94
102
|
#resolveImpitBrowser(fingerprint) {
|
|
95
103
|
if (!fingerprint?.browser)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/impit-client",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.240",
|
|
4
4
|
"description": "impit-based HTTP client implementation for Crawlee. Impersonates browser requests to avoid bot detection.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.13.0"
|
|
@@ -48,10 +48,10 @@
|
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"@apify/datastructures": "^3.0.1",
|
|
51
|
-
"@crawlee/core": "4.0.0-beta.
|
|
52
|
-
"@crawlee/http-client": "4.0.0-beta.
|
|
53
|
-
"@crawlee/types": "4.0.0-beta.
|
|
54
|
-
"impit": "^0.14.
|
|
51
|
+
"@crawlee/core": "4.0.0-beta.240",
|
|
52
|
+
"@crawlee/http-client": "4.0.0-beta.240",
|
|
53
|
+
"@crawlee/types": "4.0.0-beta.240",
|
|
54
|
+
"impit": "^0.14.6",
|
|
55
55
|
"tough-cookie": "^6.0.0"
|
|
56
56
|
},
|
|
57
57
|
"lerna": {
|
|
@@ -61,5 +61,5 @@
|
|
|
61
61
|
}
|
|
62
62
|
}
|
|
63
63
|
},
|
|
64
|
-
"gitHead": "
|
|
64
|
+
"gitHead": "040d91e9f36dfd4c99f73b63abd97b92b82d89a4"
|
|
65
65
|
}
|