@crawlee/http-client 4.0.0-beta.41 → 4.0.0-beta.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/base-http-client.d.ts.map +1 -1
- package/base-http-client.js +19 -5
- package/base-http-client.js.map +1 -1
- package/package.json +3 -3
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"base-http-client.d.ts","sourceRoot":"","sources":["../src/base-http-client.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,IAAI,uBAAuB,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAGnH,MAAM,WAAW,kBAAkB;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;;GAIG;AACH,8BAAsB,cAAe,YAAW,uBAAuB;IACnE,SAAS,CAAC,GAAG,CAAC,EAAE,aAAa,CAAC;gBAElB,OAAO,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,aAAa,CAAA;KAAE;IAIhD;;;OAGG;IACH,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,kBAAkB,GAAG,OAAO,CAAC,QAAQ,CAAC;YAEtF,YAAY;
|
|
1
|
+
{"version":3,"file":"base-http-client.d.ts","sourceRoot":"","sources":["../src/base-http-client.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,IAAI,uBAAuB,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAGnH,MAAM,WAAW,kBAAkB;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;;GAIG;AACH,8BAAsB,cAAe,YAAW,uBAAuB;IACnE,SAAS,CAAC,GAAG,CAAC,EAAE,aAAa,CAAC;gBAElB,OAAO,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,aAAa,CAAA;KAAE;IAIhD;;;OAGG;IACH,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,kBAAkB,GAAG,OAAO,CAAC,QAAQ,CAAC;YAEtF,YAAY;YAmCZ,UAAU;IAYxB,OAAO,CAAC,qBAAqB;IAW7B,OAAO,CAAC,iBAAiB;IAUzB,OAAO,CAAC,UAAU;IAKlB,OAAO,CAAC,oBAAoB;IA+B5B;;OAEG;IACG,WAAW,CAAC,cAAc,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,QAAQ,CAAC;CA8B9F"}
|
package/base-http-client.js
CHANGED
|
@@ -11,11 +11,25 @@ export class BaseHttpClient {
|
|
|
11
11
|
}
|
|
12
12
|
async applyCookies(request, cookieJar) {
|
|
13
13
|
try {
|
|
14
|
-
const
|
|
15
|
-
|
|
16
|
-
.
|
|
17
|
-
|
|
18
|
-
|
|
14
|
+
const requestCookies = request.headers.get('cookie') ?? '';
|
|
15
|
+
if (!requestCookies) {
|
|
16
|
+
// Fast path: no header cookies, use the jar directly.
|
|
17
|
+
const cookieString = await cookieJar.getCookieString(request.url);
|
|
18
|
+
if (cookieString) {
|
|
19
|
+
request.headers.set('cookie', cookieString);
|
|
20
|
+
}
|
|
21
|
+
return request;
|
|
22
|
+
}
|
|
23
|
+
// Merge jar cookies with request Cookie header. Clone the jar so we
|
|
24
|
+
// don't persist the header-only cookies into the session.
|
|
25
|
+
const merged = await cookieJar.clone();
|
|
26
|
+
await Promise.all(requestCookies
|
|
27
|
+
.split(/; */)
|
|
28
|
+
.filter(Boolean)
|
|
29
|
+
.map((pair) => merged.setCookie(pair, request.url)));
|
|
30
|
+
const cookieString = merged.getCookieStringSync(request.url);
|
|
31
|
+
if (cookieString) {
|
|
32
|
+
request.headers.set('cookie', cookieString);
|
|
19
33
|
}
|
|
20
34
|
}
|
|
21
35
|
catch (e) {
|
package/base-http-client.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"base-http-client.js","sourceRoot":"","sources":["../src/base-http-client.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAMzC;;;;GAIG;AACH,MAAM,OAAgB,cAAc;IACtB,GAAG,CAAiB;IAE9B,YAAY,OAAoC;QAC5C,IAAI,CAAC,GAAG,GAAG,OAAO,EAAE,MAAM,CAAC;IAC/B,CAAC;IAQO,KAAK,CAAC,YAAY,CAAC,OAAgB,EAAE,SAAoB;QAC7D,IAAI,CAAC;YACD,MAAM,OAAO,GAAG,CAAC,MAAM,SAAS,CAAC,
|
|
1
|
+
{"version":3,"file":"base-http-client.js","sourceRoot":"","sources":["../src/base-http-client.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAMzC;;;;GAIG;AACH,MAAM,OAAgB,cAAc;IACtB,GAAG,CAAiB;IAE9B,YAAY,OAAoC;QAC5C,IAAI,CAAC,GAAG,GAAG,OAAO,EAAE,MAAM,CAAC;IAC/B,CAAC;IAQO,KAAK,CAAC,YAAY,CAAC,OAAgB,EAAE,SAAoB;QAC7D,IAAI,CAAC;YACD,MAAM,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YAE3D,IAAI,CAAC,cAAc,EAAE,CAAC;gBAClB,sDAAsD;gBACtD,MAAM,YAAY,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBAClE,IAAI,YAAY,EAAE,CAAC;oBACf,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;gBAChD,CAAC;gBACD,OAAO,OAAO,CAAC;YACnB,CAAC;YAED,oEAAoE;YACpE,0DAA0D;YAC1D,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;YAEvC,MAAM,OAAO,CAAC,GAAG,CACb,cAAc;iBACT,KAAK,CAAC,KAAK,CAAC;iBACZ,MAAM,CAAC,OAAO,CAAC;iBACf,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAC1D,CAAC;YACF,MAAM,YAAY,GAAG,MAAM,CAAC,mBAAmB,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAE7D,IAAI,YAAY,EAAE,CAAC;gBACf,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;YAChD,CAAC;QACL,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,kCAAkC,OAAO,CAAC,GAAG,MAAO,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QACjG,CAAC;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,QAAkB,EAAE,SAAoB;QAC7D,MAAM,gBAAgB,GAAG,QAAQ,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;QAEzD,KAAK,MAAM,MAAM,IAAI,gBAAgB,EAAE,CAAC;YACpC,IAAI,CAAC;gBACD,MAAM,SAAS,CAAC,SAAS,CAAC,MAAM,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC;YACpD,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,iCAAiC,QAAQ,CAAC,GAAG,MAAO,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;YACjG,CAAC;QACL,CAAC;IACL,CAAC;IAEO,qBAAqB,CAAC,OAA4B;QAKtD,MAAM,QAAQ,GAAG,OAAO,EAAE,QAAQ,IAAI,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,CAAC;QACvE,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,OAAO,EAAE,OAAO,EAAE,SAAS,IAAI,IAAI,SAAS,EAAE,CAAC;QACvF,MAAM,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC;QAC/E,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAsB,EAAE,MAAM,EAAE,CAAC;IACnE,CAAC;IAEO,iBAAiB,CAAC,MAAoB,EAAE,aAAsB;QAClE,IAAI,MAAM,IAAI,aAAa,EAAE,CAAC;YAC1B,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;QACzE,CAAC;QACD,IAAI,MAAM,EAAE,CAAC;YACT,OAAO,MAAM,CAAC;QAClB,CAAC;QACD,OAAO,aAAa,CAAC,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC1E,CAAC;IAEO,UAAU,CAAC,QAAkB;QACjC,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;QAC/B,OAAO,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IAC/E,CAAC;IAEO,oBAAoB,CAAC,cAAuB,EAAE,QAAkB,EAAE,cAAuB;QAC7F,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAE,CAAC;QACnD,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,GAAG,IAAI,cAAc,CAAC,GAAG,CAAC,CAAC;QAEtE,MAAM,UAAU,GAAG,CAAC,cAAc,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC;QAClE,IAAI,UAAU,GAAG,UAAU,CAAC;QAC5B,IAAI,QAAQ,GAAoB,IAAI,CAAC;QAErC,IACI,QAAQ,CAAC,MAAM,KAAK,GAAG;YACvB,CAAC,CAAC,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,CAAC,IAAI,UAAU,KAAK,MAAM,CAAC,EACjF,CAAC;YACC,UAAU,GAAG,KAAK,CAAC;YACnB,QAAQ,GAAG,IAAI,CAAC;QACpB,CAAC;aAAM,CAAC;YACJ,MAAM,aAAa,GAAG,cAAc,CAAC,KAAK,EAAE,CAAC;YAC7C,QAAQ,GAAG,aAAa,CAAC,IAAI,CAAC;QAClC,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,OAAO,EAAE,CAAC;QAClC,cAAc,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;QAE5E,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE;YACnC,MAAM,EAAE,UAAU;YAClB,OAAO,EAAE,WAAW;YACpB,IAAI,EAAE,QAAQ;YACd,WAAW,EAAG,cAAsB,CAAC,WAAW;YAChD,QAAQ,EAAE,QAAQ;SACrB,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CAAC,cAAuB,EAAE,OAA4B;QACnE,MAAM,YAAY,GAAG,EAAE,CAAC;QACxB,IAAI,cAAc,GAAG,cAAc,CAAC;QACpC,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;QAC5E,cAAc,GAAG,cAAc,CAAC,KAAK,EAAE,CAAC;QAExC,OAAO,IAAI,EAAE,CAAC;YACV,MAAM,IAAI,CAAC,YAAY,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC;YAEnD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE;gBAC9C,MAAM;gBACN,QAAQ;gBACR,QAAQ,EAAE,QAAQ;aACrB,CAAC,CAAC;YAEH,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;YAE3C,IAAI,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC5B,IAAI,aAAa,EAAE,IAAI,YAAY,EAAE,CAAC;oBAClC,MAAM,IAAI,KAAK,CAAC,uBAAuB,YAAY,sBAAsB,cAAc,CAAC,GAAG,EAAE,CAAC,CAAC;gBACnG,CAAC;gBACD,cAAc,GAAG,IAAI,CAAC,oBAAoB,CAAC,cAAc,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC;gBACrF,SAAS;YACb,CAAC;YAED,OAAO,QAAQ,CAAC;QACpB,CAAC;IACL,CAAC;CACJ"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/http-client",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.43",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -47,7 +47,7 @@
|
|
|
47
47
|
"access": "public"
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
|
-
"@crawlee/types": "4.0.0-beta.
|
|
50
|
+
"@crawlee/types": "4.0.0-beta.43",
|
|
51
51
|
"tough-cookie": "^6.0.0"
|
|
52
52
|
},
|
|
53
53
|
"lerna": {
|
|
@@ -57,5 +57,5 @@
|
|
|
57
57
|
}
|
|
58
58
|
}
|
|
59
59
|
},
|
|
60
|
-
"gitHead": "
|
|
60
|
+
"gitHead": "7eb656f1c18dc8b3ef75de68e4d8d075c3abb462"
|
|
61
61
|
}
|