@crawlee/impit-client 4.0.0-beta.62 → 4.0.0-beta.63
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +7 -0
- package/index.d.ts.map +1 -1
- package/index.js +42 -1
- package/index.js.map +1 -1
- package/package.json +5 -5
package/index.d.ts
CHANGED
|
@@ -18,6 +18,12 @@ export declare class ImpitHttpClient extends BaseHttpClient {
|
|
|
18
18
|
* (and other resources) reuse.
|
|
19
19
|
*/
|
|
20
20
|
private clientCache;
|
|
21
|
+
/**
|
|
22
|
+
* Stable impit impersonation version per fingerprint object, so the same
|
|
23
|
+
* session keeps impersonating the same browser version across requests
|
|
24
|
+
* instead of rerolling on every call.
|
|
25
|
+
*/
|
|
26
|
+
private impitBrowserByFingerprint;
|
|
21
27
|
private getClient;
|
|
22
28
|
constructor(options?: Omit<ImpitOptions, 'proxyUrl' | 'timeout'> & {
|
|
23
29
|
logger?: CrawleeLogger;
|
|
@@ -26,5 +32,6 @@ export declare class ImpitHttpClient extends BaseHttpClient {
|
|
|
26
32
|
* @inheritDoc
|
|
27
33
|
*/
|
|
28
34
|
fetch(request: Request, options?: RequestInit & CustomFetchOptions): Promise<Response>;
|
|
35
|
+
private resolveImpitBrowser;
|
|
29
36
|
}
|
|
30
37
|
//# sourceMappingURL=index.d.ts.map
|
package/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC/D,OAAO,EAAE,cAAc,EAAmB,MAAM,sBAAsB,CAAC;AACvE,OAAO,KAAK,EAAE,aAAa,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC/D,OAAO,EAAE,cAAc,EAAmB,MAAM,sBAAsB,CAAC;AACvE,OAAO,KAAK,EAAE,aAAa,EAAsB,MAAM,gBAAgB,CAAC;AACxE,OAAO,EAAuC,KAAK,YAAY,EAAE,MAAM,OAAO,CAAC;AA0B/E,eAAO,MAAM,OAAO;;;CAGV,CAAC;AAEX;;GAEG;AACH,qBAAa,eAAgB,SAAQ,cAAc;IAC/C,OAAO,CAAC,YAAY,CAAe;IAEnC;;;;;OAKG;IACH,OAAO,CAAC,WAAW,CAA2F;IAE9G;;;;OAIG;IACH,OAAO,CAAC,yBAAyB,CAAmD;IAEpF,OAAO,CAAC,SAAS;gBAgBL,OAAO,CAAC,EAAE,IAAI,CAAC,YAAY,EAAE,UAAU,GAAG,SAAS,CAAC,GAAG;QAAE,MAAM,CAAC,EAAE,aAAa,CAAA;KAAE;IAK7F;;OAEG;IACG,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,WAAW,GAAG,kBAAkB,GAAG,OAAO,CAAC,QAAQ,CAAC;IAkB5F,OAAO,CAAC,mBAAmB;CAY9B"}
|
package/index.js
CHANGED
|
@@ -1,6 +1,26 @@
|
|
|
1
1
|
import { BaseHttpClient, ResponseWithUrl } from '@crawlee/http-client';
|
|
2
2
|
import { Impit } from 'impit';
|
|
3
3
|
import { LruCache } from '@apify/datastructures';
|
|
4
|
+
// Concrete impit impersonation profiles per browser family. The plain `chrome` /
|
|
5
|
+
// `firefox` aliases fall back to the oldest available version, which is a
|
|
6
|
+
// fingerprint giveaway — we pick one of these explicitly instead. Keep in sync
|
|
7
|
+
// with impit's `Browser` type when bumping the dependency.
|
|
8
|
+
const IMPIT_VERSIONS_BY_BROWSER = {
|
|
9
|
+
chrome: [
|
|
10
|
+
'chrome100',
|
|
11
|
+
'chrome101',
|
|
12
|
+
'chrome104',
|
|
13
|
+
'chrome107',
|
|
14
|
+
'chrome110',
|
|
15
|
+
'chrome116',
|
|
16
|
+
'chrome124',
|
|
17
|
+
'chrome125',
|
|
18
|
+
'chrome131',
|
|
19
|
+
'chrome136',
|
|
20
|
+
'chrome142',
|
|
21
|
+
],
|
|
22
|
+
firefox: ['firefox128', 'firefox133', 'firefox135', 'firefox144'],
|
|
23
|
+
};
|
|
4
24
|
export const Browser = {
|
|
5
25
|
'Chrome': 'chrome',
|
|
6
26
|
'Firefox': 'firefox',
|
|
@@ -17,6 +37,12 @@ export class ImpitHttpClient extends BaseHttpClient {
|
|
|
17
37
|
* (and other resources) reuse.
|
|
18
38
|
*/
|
|
19
39
|
clientCache = new LruCache({ maxLength: 10 });
|
|
40
|
+
/**
|
|
41
|
+
* Stable impit impersonation version per fingerprint object, so the same
|
|
42
|
+
* session keeps impersonating the same browser version across requests
|
|
43
|
+
* instead of rerolling on every call.
|
|
44
|
+
*/
|
|
45
|
+
impitBrowserByFingerprint = new WeakMap();
|
|
20
46
|
getClient(options) {
|
|
21
47
|
const { cookieJar, ...rest } = options;
|
|
22
48
|
const cacheKey = JSON.stringify(rest);
|
|
@@ -36,9 +62,11 @@ export class ImpitHttpClient extends BaseHttpClient {
|
|
|
36
62
|
* @inheritDoc
|
|
37
63
|
*/
|
|
38
64
|
async fetch(request, options) {
|
|
39
|
-
const { proxyUrl, redirect, signal } = options ?? {};
|
|
65
|
+
const { proxyUrl, redirect, signal, fingerprint } = options ?? {};
|
|
66
|
+
const impitBrowser = this.resolveImpitBrowser(fingerprint);
|
|
40
67
|
const impit = this.getClient({
|
|
41
68
|
...this.impitOptions,
|
|
69
|
+
...(impitBrowser ? { browser: impitBrowser } : {}),
|
|
42
70
|
proxyUrl,
|
|
43
71
|
followRedirects: redirect === 'follow',
|
|
44
72
|
});
|
|
@@ -46,5 +74,18 @@ export class ImpitHttpClient extends BaseHttpClient {
|
|
|
46
74
|
// todo - cast shouldn't be needed here, impit returns `Uint8Array`
|
|
47
75
|
return new ResponseWithUrl(response.body, response);
|
|
48
76
|
}
|
|
77
|
+
resolveImpitBrowser(fingerprint) {
|
|
78
|
+
if (!fingerprint?.browser)
|
|
79
|
+
return undefined;
|
|
80
|
+
const versions = IMPIT_VERSIONS_BY_BROWSER[fingerprint.browser];
|
|
81
|
+
if (!versions?.length)
|
|
82
|
+
return undefined;
|
|
83
|
+
const cached = this.impitBrowserByFingerprint.get(fingerprint);
|
|
84
|
+
if (cached)
|
|
85
|
+
return cached;
|
|
86
|
+
const picked = versions[Math.floor(Math.random() * versions.length)];
|
|
87
|
+
this.impitBrowserByFingerprint.set(fingerprint, picked);
|
|
88
|
+
return picked;
|
|
89
|
+
}
|
|
49
90
|
}
|
|
50
91
|
//# sourceMappingURL=index.js.map
|
package/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAEvE,OAAO,EAAE,KAAK,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAEvE,OAAO,EAAE,KAAK,EAAmD,MAAM,OAAO,CAAC;AAG/E,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAEjD,iFAAiF;AACjF,0EAA0E;AAC1E,+EAA+E;AAC/E,2DAA2D;AAC3D,MAAM,yBAAyB,GAAgF;IAC3G,MAAM,EAAE;QACJ,WAAW;QACX,WAAW;QACX,WAAW;QACX,WAAW;QACX,WAAW;QACX,WAAW;QACX,WAAW;QACX,WAAW;QACX,WAAW;QACX,WAAW;QACX,WAAW;KACd;IACD,OAAO,EAAE,CAAC,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY,CAAC;CACpE,CAAC;AAEF,MAAM,CAAC,MAAM,OAAO,GAAG;IACnB,QAAQ,EAAE,QAAQ;IAClB,SAAS,EAAE,SAAS;CACd,CAAC;AAEX;;GAEG;AACH,MAAM,OAAO,eAAgB,SAAQ,cAAc;IACvC,YAAY,CAAe;IAEnC;;;;;OAKG;IACK,WAAW,GAA2D,IAAI,QAAQ,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC;IAE9G;;;;OAIG;IACK,yBAAyB,GAAG,IAAI,OAAO,EAAoC,CAAC;IAE5E,SAAS,CAAC,OAAqB;QACnC,MAAM,EAAE,SAAS,EAAE,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC;QAEvC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,cAAc,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEtD,IAAI,cAAc,IAAI,CAAC,CAAC,SAAS,IAAI,cAAc,CAAC,SAAS,KAAK,SAAS,CAAC,EAAE,CAAC;YAC3E,OAAO,cAAc,CAAC,MAAM,CAAC;QACjC,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC;QAClC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,SAA2B,EAAE,CAAC,CAAC;QAEnF,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,YAAY,OAAiF;QACzF,KAAK,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QACnC,IAAI,CAAC,YAAY,GAAG,OAAO,IAAI,EAAE,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK,CAAC,OAAgB,EAAE,OAA0C;QACpE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,OAAO,IAAI,EAAE,CAAC;QAElE,MAAM,YAAY,GAAG,IAAI,CAAC,mBAAmB,CAAC,WAAW,CAAC,CAAC;QAE3D,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC;YACzB,GAAG,IAAI,CAAC,YAAY;YACpB,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAClD,QAAQ;YACR,eAAe,EAAE,QAAQ,KAAK,QAAQ;SACzC,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,MAAM,EAAE,MAAM,IAAI,SAAS,EAAE,CAAC,CAAC;QAE7E,mEAAmE;QACnE,OAAO,IAAI,eAAe,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IACxD,CAAC;IAEO,mBAAmB,CAAC,WAAgC;QACxD,IAAI,CAAC,WAAW,EAAE,OAAO;YAAE,OAAO,SAAS,CAAC;QAC5C,MAAM,QAAQ,GAAG,yBAAyB,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;QAChE,IAAI,CAAC,QAAQ,EAAE,MAAM;YAAE,OAAO,SAAS,CAAC;QAExC,MAAM,MAAM,GAAG,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QAC/D,IAAI,MAAM;YAAE,OAAO,MAAM,CAAC;QAE1B,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;QACrE,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;QACxD,OAAO,MAAM,CAAC;IAClB,CAAC;CACJ"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/impit-client",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.63",
|
|
4
4
|
"description": "impit-based HTTP client implementation for Crawlee. Impersonates browser requests to avoid bot detection.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -48,9 +48,9 @@
|
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"@apify/datastructures": "^2.0.3",
|
|
51
|
-
"@crawlee/http-client": "4.0.0-beta.
|
|
52
|
-
"@crawlee/types": "4.0.0-beta.
|
|
53
|
-
"impit": "^0.
|
|
51
|
+
"@crawlee/http-client": "4.0.0-beta.63",
|
|
52
|
+
"@crawlee/types": "4.0.0-beta.63",
|
|
53
|
+
"impit": "^0.14.1",
|
|
54
54
|
"tough-cookie": "^6.0.0"
|
|
55
55
|
},
|
|
56
56
|
"lerna": {
|
|
@@ -60,5 +60,5 @@
|
|
|
60
60
|
}
|
|
61
61
|
}
|
|
62
62
|
},
|
|
63
|
-
"gitHead": "
|
|
63
|
+
"gitHead": "6255e805b2d2ae85024897a2738315ae04df4424"
|
|
64
64
|
}
|