@crawlee/core 4.0.0-beta.21 → 4.0.0-beta.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"got-scraping-http-client.d.ts","sourceRoot":"","sources":["../../src/http_clients/got-scraping-http-client.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAMxF;;GAEG;AACH,qBAAa,qBAAsB,YAAW,cAAc;IACxD;;;OAGG;IACH,OAAO,CAAC,eAAe;IAMvB,OAAO,CAAE,cAAc;IAmBvB,OAAO,CAAC,YAAY;IAIpB;;OAEG;IACG,WAAW,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,QAAQ,CAAC;
|
|
1
|
+
{"version":3,"file":"got-scraping-http-client.d.ts","sourceRoot":"","sources":["../../src/http_clients/got-scraping-http-client.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAMxF;;GAEG;AACH,qBAAa,qBAAsB,YAAW,cAAc;IACxD;;;OAGG;IACH,OAAO,CAAC,eAAe;IAMvB,OAAO,CAAE,cAAc;IAmBvB,OAAO,CAAC,YAAY;IAIpB;;OAEG;IACG,WAAW,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,QAAQ,CAAC;IA2BpF;;OAEG;IACG,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,QAAQ,CAAC;CA8D7E"}
|
|
@@ -35,7 +35,7 @@ export class GotScrapingHttpClient {
|
|
|
35
35
|
* @inheritDoc
|
|
36
36
|
*/
|
|
37
37
|
async sendRequest(request, options) {
|
|
38
|
-
const { session, timeout, proxyUrl } = options ?? {};
|
|
38
|
+
const { session, timeout, cookieJar, proxyUrl } = options ?? {};
|
|
39
39
|
if (!this.validateRequest(request)) {
|
|
40
40
|
throw new Error(`The HTTP method CONNECT is not supported by the GotScrapingHttpClient.`);
|
|
41
41
|
}
|
|
@@ -46,10 +46,7 @@ export class GotScrapingHttpClient {
|
|
|
46
46
|
body: request.body ? Readable.fromWeb(request.body) : undefined,
|
|
47
47
|
proxyUrl: proxyUrl ?? session?.proxyInfo?.url,
|
|
48
48
|
timeout: { request: timeout },
|
|
49
|
-
|
|
50
|
-
// `HttpCrawler` reads the cookies beforehand and sets them in `request.headers`.
|
|
51
|
-
// Using the `cookieJar` option directly would override that.
|
|
52
|
-
cookieJar: undefined,
|
|
49
|
+
cookieJar,
|
|
53
50
|
});
|
|
54
51
|
const responseHeaders = this.parseHeaders(gotResult.headers);
|
|
55
52
|
return new ResponseWithUrl(new Uint8Array(gotResult.rawBody), {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"got-scraping-http-client.js","sourceRoot":"","sources":["../../src/http_clients/got-scraping-http-client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAIvC,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAE3C,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD;;GAEG;AACH,MAAM,OAAO,qBAAqB;IAC9B;;;OAGG;IACK,eAAe,CACnB,OAAgB;QAEhB,OAAO,CAAC,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAO,CAAC,CAAC;IAC7D,CAAC;IAEO,CAAC,cAAc,CACnB,OAAsD;QAEtD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YACjD,4BAA4B;YAC5B,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC7C,SAAS;YACb,CAAC;YAED,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;gBACvB,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;oBACpB,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;gBACnB,CAAC;YACL,CAAC;iBAAM,CAAC;gBACJ,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YACvB,CAAC;QACL,CAAC;IACL,CAAC;IAEO,YAAY,CAAC,OAAsD;QACvE,OAAO,IAAI,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAC1D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CAAC,OAAgB,EAAE,OAA4B;QAC5D,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,OAAO,IAAI,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"got-scraping-http-client.js","sourceRoot":"","sources":["../../src/http_clients/got-scraping-http-client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAIvC,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAE3C,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD;;GAEG;AACH,MAAM,OAAO,qBAAqB;IAC9B;;;OAGG;IACK,eAAe,CACnB,OAAgB;QAEhB,OAAO,CAAC,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAO,CAAC,CAAC;IAC7D,CAAC;IAEO,CAAC,cAAc,CACnB,OAAsD;QAEtD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YACjD,4BAA4B;YAC5B,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC7C,SAAS;YACb,CAAC;YAED,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;gBACvB,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;oBACpB,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;gBACnB,CAAC;YACL,CAAC;iBAAM,CAAC;gBACJ,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YACvB,CAAC;QACL,CAAC;IACL,CAAC;IAEO,YAAY,CAAC,OAAsD;QACvE,OAAO,IAAI,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAC1D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CAAC,OAAgB,EAAE,OAA4B;QAC5D,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,OAAO,IAAI,EAAE,CAAC;QAEhE,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CAAC,wEAAwE,CAAC,CAAC;QAC9F,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC;YAChC,GAAG,EAAE,OAAO,CAAC,GAAI;YACjB,MAAM,EAAE,OAAO,CAAC,MAA2B;YAC3C,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YACtD,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,IAAW,CAAC,CAAC,CAAC,CAAC,SAAS;YACtE,QAAQ,EAAE,QAAQ,IAAI,OAAO,EAAE,SAAS,EAAE,GAAG;YAC7C,OAAO,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE;YAC7B,SAAS;SACZ,CAAC,CAAC;QAEH,MAAM,eAAe,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAE7D,OAAO,IAAI,eAAe,CAAC,IAAI,UAAU,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE;YAC1D,OAAO,EAAE,eAAe;YACxB,MAAM,EAAE,SAAS,CAAC,UAAU;YAC5B,UAAU,EAAE,SAAS,CAAC,aAAa,IAAI,EAAE;YACzC,GAAG,EAAE,SAAS,CAAC,GAAG;SACrB,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,OAAgB,EAAE,OAAuB;QAClD,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,OAAO,IAAI,EAAE,CAAC;QAE3C,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CAAC,wEAAwE,CAAC,CAAC;QAC9F,CAAC;QACD,qDAAqD;QACrD,OAAO,IAAI,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE;YACzC,MAAM,MAAM,GAAG,WAAW,CAAC;gBACvB,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,MAAM,EAAE,OAAO,CAAC,MAA2B;gBAC3C,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;gBACtD,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,IAAW,CAAC,CAAC,CAAC,CAAC,SAAS;gBACtE,QAAQ,EAAE,IAAI;gBACd,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG;gBACjC,OAAO,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE;gBAC7B,SAAS,EAAE,SAAS;aACvB,CAAC,CAAC;YAEH,MAAM,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,cAAuB,EAAE,gBAAqB,EAAE,EAAE;gBACrE,MAAM,sBAAsB,GAAG,IAAI,eAAe,CAAC,gBAAgB,CAAC,OAAO,EAAE;oBACzE,OAAO,EAAE,gBAAgB,CAAC,OAAO;oBACjC,MAAM,EAAE,gBAAgB,CAAC,UAAU;oBACnC,UAAU,EAAE,gBAAgB,CAAC,aAAa;oBAC1C,GAAG,EAAE,gBAAgB,CAAC,GAAG;iBAC5B,CAAC,CAAC;gBAEH,MAAM,aAAa,GAAG,IAAI,OAAO,CAC7B,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,OAAO,CAAC;qBACjC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;qBAC3F,IAAI,EAAwB,CACpC,CAAC;gBAEF,OAAO,EAAE,UAAU,EAAE,CAAC,sBAAsB,EAAE;oBAC1C,GAAG,EAAE,cAAc,CAAC,GAAG;oBACvB,OAAO,EAAE,aAAa;iBACzB,CAAC,CAAC;gBAEH,cAAc,CAAC,OAAO,GAAG,MAAM,CAAC,WAAW,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC;YACzE,CAAC,CAAC,CAAC;YAEH,yEAAyE;YACzE,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAClE,MAAM,CAAC,GAAG,EAAE,CAAC;YACjB,CAAC;YAED,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAE3B,MAAM,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,QAAuB,EAAE,EAAE;gBAC9C,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;gBACpD,qFAAqF;gBACrF,OAAO,CACH,IAAI,eAAe,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAQ,EAAE;oBAC/C,MAAM,EAAE,QAAQ,CAAC,UAAU;oBAC3B,UAAU,EAAE,QAAQ,CAAC,aAAa,IAAI,EAAE;oBACxC,OAAO;oBACP,GAAG,EAAE,QAAQ,CAAC,GAAG;iBACpB,CAAC,CACL,CAAC;YACN,CAAC,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;IACP,CAAC;CACJ"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/core",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.22",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -53,9 +53,9 @@
|
|
|
53
53
|
"@apify/pseudo_url": "^2.0.59",
|
|
54
54
|
"@apify/timeout": "^0.3.2",
|
|
55
55
|
"@apify/utilities": "^2.15.5",
|
|
56
|
-
"@crawlee/memory-storage": "4.0.0-beta.
|
|
57
|
-
"@crawlee/types": "4.0.0-beta.
|
|
58
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
56
|
+
"@crawlee/memory-storage": "4.0.0-beta.22",
|
|
57
|
+
"@crawlee/types": "4.0.0-beta.22",
|
|
58
|
+
"@crawlee/utils": "4.0.0-beta.22",
|
|
59
59
|
"@sapphire/async-queue": "^1.5.5",
|
|
60
60
|
"@vladfrangu/async_event_emitter": "^2.4.6",
|
|
61
61
|
"csv-stringify": "^6.5.2",
|
|
@@ -77,5 +77,5 @@
|
|
|
77
77
|
}
|
|
78
78
|
}
|
|
79
79
|
},
|
|
80
|
-
"gitHead": "
|
|
80
|
+
"gitHead": "74e5a8dae2b5c919ca40cc3f9135744448aca0ac"
|
|
81
81
|
}
|