@crawlee/core 4.0.0-beta.21 → 4.0.0-beta.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/crawlers/crawler_commons.d.ts +3 -6
- package/crawlers/crawler_commons.d.ts.map +1 -1
- package/crawlers/crawler_commons.js.map +1 -1
- package/index.d.ts +0 -1
- package/index.d.ts.map +1 -1
- package/index.js +0 -1
- package/index.js.map +1 -1
- package/package.json +5 -6
- package/http_clients/base-http-client.d.ts +0 -12
- package/http_clients/base-http-client.d.ts.map +0 -1
- package/http_clients/base-http-client.js +0 -44
- package/http_clients/base-http-client.js.map +0 -1
- package/http_clients/form-data-like.d.ts +0 -67
- package/http_clients/form-data-like.d.ts.map +0 -1
- package/http_clients/form-data-like.js +0 -5
- package/http_clients/form-data-like.js.map +0 -1
- package/http_clients/got-scraping-http-client.d.ts +0 -22
- package/http_clients/got-scraping-http-client.d.ts.map +0 -1
- package/http_clients/got-scraping-http-client.js +0 -116
- package/http_clients/got-scraping-http-client.js.map +0 -1
- package/http_clients/index.d.ts +0 -3
- package/http_clients/index.d.ts.map +0 -1
- package/http_clients/index.js +0 -3
- package/http_clients/index.js.map +0 -1
|
@@ -1,6 +1,4 @@
|
|
|
1
|
-
import type { Dictionary, ProxyInfo } from '@crawlee/types';
|
|
2
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
3
|
-
import type { OptionsInit } from 'got-scraping';
|
|
1
|
+
import type { Dictionary, HttpRequestOptions, ProxyInfo, SendRequestOptions } from '@crawlee/types';
|
|
4
2
|
import type { ReadonlyDeep, SetRequired } from 'type-fest';
|
|
5
3
|
import type { Configuration } from '../configuration.js';
|
|
6
4
|
import type { EnqueueLinksOptions } from '../enqueue_links/enqueue_links.js';
|
|
@@ -114,8 +112,7 @@ export interface CrawlingContext<UserData extends Dictionary = Dictionary> exten
|
|
|
114
112
|
*/
|
|
115
113
|
enqueueLinks(options: ReadonlyDeep<Omit<SetRequired<EnqueueLinksOptions, 'urls'>, 'requestQueue' | 'robotsTxtFile'>> & Pick<EnqueueLinksOptions, 'requestQueue' | 'robotsTxtFile'>): Promise<unknown>;
|
|
116
114
|
/**
|
|
117
|
-
* Fires HTTP request via
|
|
118
|
-
* options on the fly.
|
|
115
|
+
* Fires HTTP request via the internal HTTP client, allowing to override the request options on the fly.
|
|
119
116
|
*
|
|
120
117
|
* This is handy when you work with a browser crawler but want to execute some requests outside it (e.g. API requests).
|
|
121
118
|
* Check the [Skipping navigations for certain requests](https://crawlee.dev/js/docs/examples/skip-navigation) example for
|
|
@@ -130,7 +127,7 @@ export interface CrawlingContext<UserData extends Dictionary = Dictionary> exten
|
|
|
130
127
|
* },
|
|
131
128
|
* ```
|
|
132
129
|
*/
|
|
133
|
-
sendRequest(
|
|
130
|
+
sendRequest: (requestOverrides?: Partial<HttpRequestOptions>, optionsOverrides?: SendRequestOptions) => Promise<Response>;
|
|
134
131
|
/**
|
|
135
132
|
* Register a function to be called at the very end of the request handling process. This is useful for resources that should be accessible to error handlers, for instance.
|
|
136
133
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crawler_commons.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_commons.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,
|
|
1
|
+
{"version":3,"file":"crawler_commons.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_commons.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,kBAAkB,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AACpG,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAE3D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,mCAAmC,CAAC;AAC7E,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,KAAK,aAAa,EAAE,MAAM,gCAAgC,CAAC;AACnF,OAAO,KAAK,EAAE,4BAA4B,EAAE,MAAM,iCAAiC,CAAC;AAEpF,gBAAgB;AAChB,MAAM,MAAM,KAAK,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,IAAI,GAAG,KAAK,CAAC;AAEtD,gBAAgB;AAChB,MAAM,MAAM,YAAY,CAAC,CAAC,EAAE,CAAC,SAAS,MAAM,CAAC,IAAI,CAAC,GAAG;KAAG,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;CAAE,CAAC;AAE1E,MAAM,MAAM,aAAa,CAAC,CAAC,SAAS,OAAO,IAAI,YAAY,CAAC,CAAC,EAAE,IAAI,GAAG,WAAW,CAAC,CAAC;AAEnF,gBAAgB;AAChB,MAAM,MAAM,aAAa,CAAC,OAAO,SAAS,yBAAyB,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,IAAI,GAC5F,OAAO,GACP;IACI,OAAO,EAAE,aAAa,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;CAC9C,GAAG,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;AAEnC,MAAM,WAAW,yBAAyB,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU;IAC/E,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;OAGG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;IAEtB;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IAE3B;;;;;;OAMG;IACH,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC,UAAU,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,eAAe,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE1G;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,YAAY,EAAE,CACV,OAAO,EAAE,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,mBAAmB,EAAE,MAAM,CAAC,EAAE,cAAc,GAAG,eAAe,CAAC,CAAC,KACtG,OAAO,CAAC,OAAO,CAAC,CAAC;IAEtB;;;;;OAKG;IACH,WAAW,EAAE,CACT,YAAY,EAAE,YAAY,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC,EAC/C,OAAO,CAAC,EAAE,YAAY,CAAC,4BAA4B,CAAC,KACnD,OAAO,CAAC,IAAI,CAAC,CAAC;IAEnB;;OAEG;IACH,QAAQ,EAAE,CAAC,KAAK,SAAS,UAAU,GAAG,UAAU,EAAE,YAAY,CAAC,EAAE,KAAK,KAAK,OAAO,CAAC,KAAK,CAAC,CAAC;IAE1F;;OAEG;IACH,gBAAgB,EAAE,CACd,QAAQ,CAAC,EAAE,MAAM,KAChB,OAAO,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,GAAG,MAAM,GAAG,UAAU,GAAG,mBAAmB,GAAG,UAAU,GAAG,cAAc,CAAC,CAAC,CAAC;IAElH;;OAEG;IACH,GAAG,EAAE,GAAG,CAAC;CACZ;AAED,MAAM,WAAW,eAAe,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU,CAAE,SAAQ,yBAAyB,CAAC,QAAQ,CAAC;IAClH;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACH,YAAY,CACR,OAAO,EAAE,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,mBAAmB,EAAE,MAAM,CAAC,EAAE,cAAc,GAAG,eAAe,CAAC,CAAC,GACnG,IAAI,CAAC,mBAAmB,EAAE,cAAc,GAAG,eAAe,CAAC,GAChE,OAAO,CAAC,OAAO,CAAC,CAAC;IAEpB;;;;;;;;;;;;;;;OAeG;IACH,WAAW,EAAE,CACT,gBAAgB,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC,EAC9C,gBAAgB,CAAC,EAAE,kBAAkB,KACpC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAEvB;;OAEG;IACH,uBAAuB,CAAC,OAAO,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC;CAClE;AAED;;;;GAIG;AACH,qBAAa,oBAAoB;IASzB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,eAAe;IAT3B,OAAO,CAAC,qBAAqB,CACtB;IAEP,OAAO,CAAC,aAAa,CAA2D;IAEhF,OAAO,CAAC,gBAAgB,CAA8D;gBAG1E,MAAM,EAAE,aAAa,EACrB,eAAe,EAAE,MAAM;IAGnC;;OAEG;IACH,IAAI,KAAK,IAAI,YAAY,CAAC;QACtB,QAAQ,EAAE,UAAU,CAAC,yBAAyB,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;QAC9D,WAAW,EAAE,UAAU,CAAC,yBAAyB,CAAC,aAAa,CAAC,CAAC,EAAE,CAAC;KACvE,CAAC,CAKD;IAED;;OAEG;IACH,IAAI,oBAAoB,IAAI,YAAY,CACpC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,YAAY,EAAE,OAAO,CAAC;QAAC,OAAO,CAAC,EAAE,aAAa,CAAA;KAAE,CAAC,CAAC,CACrF,CAEA;IAED;;OAEG;IACH,IAAI,YAAY,IAAI,YAAY,CAAC;QAAE,IAAI,EAAE,UAAU,CAAC;QAAC,eAAe,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC,CAIjF;IAED;;OAEG;IACH,IAAI,YAAY,IAAI,YAAY,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC,CAkBlE;IAED;;OAEG;IACH,IAAI,gBAAgB,IAAI,YAAY,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC,CAgB1E;IAED,QAAQ,EAAE,yBAAyB,CAAC,UAAU,CAAC,CAE7C;IAEF,WAAW,EAAE,yBAAyB,CAAC,aAAa,CAAC,CAEnD;IAEF,QAAQ,EAAE,yBAAyB,CAAC,UAAU,CAAC,CAG7C;IAEF,gBAAgB,EAAE,yBAAyB,CAAC,kBAAkB,CAAC,CAa7D;IAEF,OAAO,CAAC,WAAW,CAAwF;IAE3G,OAAO,CAAC,4BAA4B,CAIlC;IAEF,OAAO,CAAC,4BAA4B,CASlC;CACL"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crawler_commons.js","sourceRoot":"","sources":["../../src/crawlers/crawler_commons.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"crawler_commons.js","sourceRoot":"","sources":["../../src/crawlers/crawler_commons.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,aAAa,EAAsB,MAAM,gCAAgC,CAAC;AA6JnF;;;;GAIG;AACH,MAAM,OAAO,oBAAoB;IASjB;IACA;IATJ,qBAAqB,GACzB,EAAE,CAAC;IAEC,aAAa,GAAwD,EAAE,CAAC;IAExE,gBAAgB,GAA2D,EAAE,CAAC;IAEtF,YACY,MAAqB,EACrB,eAAuB;QADvB,WAAM,GAAN,MAAM,CAAe;QACrB,oBAAe,GAAf,eAAe,CAAQ;IAChC,CAAC;IAEJ;;OAEG;IACH,IAAI,KAAK;QAIL,OAAO;YACH,QAAQ,EAAE,IAAI,CAAC,aAAa;YAC5B,WAAW,EAAE,IAAI,CAAC,gBAAgB;SACrC,CAAC;IACN,CAAC;IAED;;OAEG;IACH,IAAI,oBAAoB;QAGpB,OAAO,IAAI,CAAC,qBAAqB,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,IAAI,YAAY;QACZ,OAAO,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,eAAe,CAAC,EAAE,EAAE,CAC1D,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC,CACnF,CAAC;IACN,CAAC;IAED;;OAEG;IACH,IAAI,YAAY;QACZ,MAAM,MAAM,GAAsC,EAAE,CAAC;QAErD,KAAK,MAAM,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC7C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC7B,IACI,OAAO,OAAO,KAAK,QAAQ;oBAC3B,CAAC,CAAC,CAAC,iBAAiB,IAAI,OAAO,CAAC,IAAI,OAAO,CAAC,eAAe,KAAK,SAAS,CAAC;oBAC1E,OAAO,CAAC,GAAG,KAAK,SAAS,EAC3B,CAAC;oBACC,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;gBAC5D,CAAC;qBAAM,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;oBACrC,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC;gBAClC,CAAC;YACL,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;OAEG;IACH,IAAI,gBAAgB;QAChB,MAAM,MAAM,GAA0C,EAAE,CAAC;QAEzD,KAAK,MAAM,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC7C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC7B,IACI,OAAO,OAAO,KAAK,QAAQ;oBAC3B,iBAAiB,IAAI,OAAO;oBAC5B,OAAO,CAAC,eAAe,KAAK,SAAS,EACvC,CAAC;oBACC,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,eAAe,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;gBAC5E,CAAC;YACL,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,QAAQ,GAA0C,KAAK,EAAE,IAAI,EAAE,eAAe,EAAE,EAAE;QAC9E,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC,CAAC;IACrD,CAAC,CAAC;IAEF,WAAW,GAA6C,KAAK,EAAE,QAAQ,EAAE,OAAO,GAAG,EAAE,EAAE,EAAE;QACrF,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;IACpD,CAAC,CAAC;IAEF,QAAQ,GAA0C,KAAK,EAAE,YAAY,EAAE,EAAE;QACrE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;QACrD,OAAO,MAAM,KAAK,CAAC,iBAAiB,CAAC,IAAI,CAAC,eAAe,EAAE,YAAY,CAAC,CAAC;IAC7E,CAAC,CAAC;IAEF,gBAAgB,GAAkD,KAAK,EAAE,QAAQ,EAAE,EAAE;QACjF,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAE1E,OAAO;YACH,EAAE,EAAE,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC;YAC9B,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,4BAA4B,CAAC,QAAQ,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;YACxG,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;gBACpC,IAAI,CAAC,4BAA4B,CAAC,QAAQ,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;YACrE,CAAC;YACD,iBAAiB,EAAE,KAAK,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC;YACtD,YAAY,EAAE,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC;SAC/C,CAAC;IACN,CAAC,CAAC;IAEM,WAAW,GAAG,CAAC,QAAiB,EAAU,EAAE,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;IAEnG,4BAA4B,GAAG,CAAC,QAA4B,EAAE,GAAW,EAAE,EAAE;QACjF,MAAM,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;QACtC,IAAI,CAAC,qBAAqB,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;QACtC,OAAO,IAAI,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,YAAY,IAAI,IAAI,CAAC;IACpE,CAAC,CAAC;IAEM,4BAA4B,GAAG,CACnC,QAA4B,EAC5B,GAAW,EACX,YAAqB,EACrB,OAAuB,EACzB,EAAE;QACA,MAAM,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;QACtC,IAAI,CAAC,qBAAqB,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;QACtC,IAAI,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC;IACpE,CAAC,CAAC;CACL"}
|
package/index.d.ts
CHANGED
|
@@ -4,7 +4,6 @@ export * from './configuration.js';
|
|
|
4
4
|
export * from './crawlers/index.js';
|
|
5
5
|
export * from './enqueue_links/index.js';
|
|
6
6
|
export * from './events/index.js';
|
|
7
|
-
export * from './http_clients/index.js';
|
|
8
7
|
export * from './log.js';
|
|
9
8
|
export * from './proxy_configuration.js';
|
|
10
9
|
export * from './request.js';
|
package/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC;AACnC,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,mBAAmB,CAAC;AAClC,cAAc,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC;AACnC,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,mBAAmB,CAAC;AAClC,cAAc,UAAU,CAAC;AACzB,cAAc,0BAA0B,CAAC;AACzC,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,oBAAoB,CAAC;AACnC,cAAc,yBAAyB,CAAC;AACxC,cAAc,qBAAqB,CAAC;AACpC,cAAc,iBAAiB,CAAC;AAChC,cAAc,mBAAmB,CAAC;AAClC,cAAc,wBAAwB,CAAC;AACvC,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC"}
|
package/index.js
CHANGED
|
@@ -4,7 +4,6 @@ export * from './configuration.js';
|
|
|
4
4
|
export * from './crawlers/index.js';
|
|
5
5
|
export * from './enqueue_links/index.js';
|
|
6
6
|
export * from './events/index.js';
|
|
7
|
-
export * from './http_clients/index.js';
|
|
8
7
|
export * from './log.js';
|
|
9
8
|
export * from './proxy_configuration.js';
|
|
10
9
|
export * from './request.js';
|
package/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC;AACnC,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,mBAAmB,CAAC;AAClC,cAAc,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC;AACnC,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,mBAAmB,CAAC;AAClC,cAAc,UAAU,CAAC;AACzB,cAAc,0BAA0B,CAAC;AACzC,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,oBAAoB,CAAC;AACnC,cAAc,yBAAyB,CAAC;AACxC,cAAc,qBAAqB,CAAC;AACpC,cAAc,iBAAiB,CAAC;AAChC,cAAc,mBAAmB,CAAC;AAClC,cAAc,wBAAwB,CAAC;AACvC,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/core",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.23",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -53,14 +53,13 @@
|
|
|
53
53
|
"@apify/pseudo_url": "^2.0.59",
|
|
54
54
|
"@apify/timeout": "^0.3.2",
|
|
55
55
|
"@apify/utilities": "^2.15.5",
|
|
56
|
-
"@crawlee/memory-storage": "4.0.0-beta.
|
|
57
|
-
"@crawlee/types": "4.0.0-beta.
|
|
58
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
56
|
+
"@crawlee/memory-storage": "4.0.0-beta.23",
|
|
57
|
+
"@crawlee/types": "4.0.0-beta.23",
|
|
58
|
+
"@crawlee/utils": "4.0.0-beta.23",
|
|
59
59
|
"@sapphire/async-queue": "^1.5.5",
|
|
60
60
|
"@vladfrangu/async_event_emitter": "^2.4.6",
|
|
61
61
|
"csv-stringify": "^6.5.2",
|
|
62
62
|
"fs-extra": "^11.3.0",
|
|
63
|
-
"got-scraping": "^4.1.1",
|
|
64
63
|
"json5": "^2.2.3",
|
|
65
64
|
"minimatch": "^10.0.1",
|
|
66
65
|
"ow": "^2.0.0",
|
|
@@ -77,5 +76,5 @@
|
|
|
77
76
|
}
|
|
78
77
|
}
|
|
79
78
|
},
|
|
80
|
-
"gitHead": "
|
|
79
|
+
"gitHead": "929fd9374b74ee5e9720b3864b39326f014cd45b"
|
|
81
80
|
}
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
import type { HttpRequest, HttpRequestOptions, IResponseWithUrl } from '@crawlee/types';
|
|
2
|
-
export declare class ResponseWithUrl extends Response implements IResponseWithUrl {
|
|
3
|
-
url: string;
|
|
4
|
-
constructor(body: BodyInit | null, init: ResponseInit & {
|
|
5
|
-
url?: string;
|
|
6
|
-
});
|
|
7
|
-
}
|
|
8
|
-
/**
|
|
9
|
-
* Converts {@link HttpRequestOptions} to a {@link HttpRequest}.
|
|
10
|
-
*/
|
|
11
|
-
export declare function processHttpRequestOptions({ searchParams, form, json, username, password, ...request }: HttpRequestOptions): HttpRequest;
|
|
12
|
-
//# sourceMappingURL=base-http-client.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"base-http-client.d.ts","sourceRoot":"","sources":["../../src/http_clients/base-http-client.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,WAAW,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAGxF,qBAAa,eAAgB,SAAQ,QAAS,YAAW,gBAAgB;IAC5D,GAAG,EAAE,MAAM,CAAC;gBACT,IAAI,EAAE,QAAQ,GAAG,IAAI,EAAE,IAAI,EAAE,YAAY,GAAG;QAAE,GAAG,CAAC,EAAE,MAAM,CAAA;KAAE;CAI3E;AAED;;GAEG;AACH,wBAAgB,yBAAyB,CAAC,EACtC,YAAY,EACZ,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,GAAG,OAAO,EACb,EAAE,kBAAkB,GAAG,WAAW,CAwClC"}
|
|
@@ -1,44 +0,0 @@
|
|
|
1
|
-
import { Readable } from 'node:stream';
|
|
2
|
-
import { applySearchParams } from '@crawlee/utils';
|
|
3
|
-
export class ResponseWithUrl extends Response {
|
|
4
|
-
url;
|
|
5
|
-
constructor(body, init) {
|
|
6
|
-
super(body, init);
|
|
7
|
-
this.url = init.url ?? '';
|
|
8
|
-
}
|
|
9
|
-
}
|
|
10
|
-
/**
|
|
11
|
-
* Converts {@link HttpRequestOptions} to a {@link HttpRequest}.
|
|
12
|
-
*/
|
|
13
|
-
export function processHttpRequestOptions({ searchParams, form, json, username, password, ...request }) {
|
|
14
|
-
const url = new URL(request.url);
|
|
15
|
-
const headers = new Headers(request.headers);
|
|
16
|
-
applySearchParams(url, searchParams);
|
|
17
|
-
if ([request.body, form, json].filter((value) => value !== undefined).length > 1) {
|
|
18
|
-
throw new Error('At most one of `body`, `form` and `json` may be specified in sendRequest arguments');
|
|
19
|
-
}
|
|
20
|
-
const body = (() => {
|
|
21
|
-
if (form !== undefined) {
|
|
22
|
-
return Readable.from(new URLSearchParams(form).toString());
|
|
23
|
-
}
|
|
24
|
-
if (json !== undefined) {
|
|
25
|
-
return Readable.from(JSON.stringify(json));
|
|
26
|
-
}
|
|
27
|
-
if (request.body !== undefined) {
|
|
28
|
-
return Readable.from(request.body);
|
|
29
|
-
}
|
|
30
|
-
return undefined;
|
|
31
|
-
})();
|
|
32
|
-
if (form !== undefined && !headers.has('content-type')) {
|
|
33
|
-
headers.set('content-type', 'application/x-www-form-urlencoded');
|
|
34
|
-
}
|
|
35
|
-
if (json !== undefined && !headers.has('content-type')) {
|
|
36
|
-
headers.set('content-type', 'application/json');
|
|
37
|
-
}
|
|
38
|
-
if (username !== undefined || password !== undefined) {
|
|
39
|
-
const encodedAuth = Buffer.from(`${username ?? ''}:${password ?? ''}`).toString('base64');
|
|
40
|
-
headers.set('authorization', `Basic ${encodedAuth}`);
|
|
41
|
-
}
|
|
42
|
-
return { ...request, body, url, headers };
|
|
43
|
-
}
|
|
44
|
-
//# sourceMappingURL=base-http-client.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"base-http-client.js","sourceRoot":"","sources":["../../src/http_clients/base-http-client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAGvC,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAEnD,MAAM,OAAO,eAAgB,SAAQ,QAAQ;IAChC,GAAG,CAAS;IACrB,YAAY,IAAqB,EAAE,IAAqC;QACpE,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAClB,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC;IAC9B,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,UAAU,yBAAyB,CAAC,EACtC,YAAY,EACZ,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,GAAG,OAAO,EACO;IACjB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACjC,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAE7C,iBAAiB,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IAErC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/E,MAAM,IAAI,KAAK,CAAC,oFAAoF,CAAC,CAAC;IAC1G,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,GAAG,EAAE;QACf,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACrB,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC/D,CAAC;QAED,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACrB,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/C,CAAC;QAED,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC7B,OAAO,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACvC,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC,CAAC,EAAE,CAAC;IAEL,IAAI,IAAI,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,mCAAmC,CAAC,CAAC;IACrE,CAAC;IAED,IAAI,IAAI,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,kBAAkB,CAAC,CAAC;IACpD,CAAC;IAED,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QACnD,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,IAAI,EAAE,IAAI,QAAQ,IAAI,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC1F,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,SAAS,WAAW,EAAE,CAAC,CAAC;IACzD,CAAC;IAED,OAAO,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC;AAC9C,CAAC"}
|
|
@@ -1,67 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* This is copied from https://github.com/octet-stream/form-data-encoder
|
|
3
|
-
*/
|
|
4
|
-
interface FileLike {
|
|
5
|
-
/**
|
|
6
|
-
* Name of the file referenced by the File object.
|
|
7
|
-
*/
|
|
8
|
-
readonly name: string;
|
|
9
|
-
/**
|
|
10
|
-
* Returns the media type ([`MIME`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types)) of the file represented by a `File` object.
|
|
11
|
-
*/
|
|
12
|
-
readonly type: string;
|
|
13
|
-
/**
|
|
14
|
-
* Size of the file parts in bytes
|
|
15
|
-
*/
|
|
16
|
-
readonly size: number;
|
|
17
|
-
/**
|
|
18
|
-
* The last modified date of the file as the number of milliseconds since the Unix epoch (January 1, 1970 at midnight). Files without a known last modified date return the current date.
|
|
19
|
-
*/
|
|
20
|
-
readonly lastModified: number;
|
|
21
|
-
/**
|
|
22
|
-
* Returns a [`ReadableStream`](https://developer.mozilla.org/en-US/docs/Web/API/ReadableStream) which upon reading returns the data contained within the [`File`](https://developer.mozilla.org/en-US/docs/Web/API/File).
|
|
23
|
-
*/
|
|
24
|
-
stream(): ReadableStream<Uint8Array> | AsyncIterable<Uint8Array>;
|
|
25
|
-
readonly [Symbol.toStringTag]?: string;
|
|
26
|
-
}
|
|
27
|
-
/**
|
|
28
|
-
* A `string` or `File` that represents a single value from a set of `FormData` key-value pairs.
|
|
29
|
-
*/
|
|
30
|
-
type FormDataEntryValue = string | FileLike;
|
|
31
|
-
/**
|
|
32
|
-
* This interface reflects minimal shape of the FormData
|
|
33
|
-
*/
|
|
34
|
-
export interface FormDataLike {
|
|
35
|
-
/**
|
|
36
|
-
* Appends a new value onto an existing key inside a FormData object,
|
|
37
|
-
* or adds the key if it does not already exist.
|
|
38
|
-
*
|
|
39
|
-
* The difference between `set()` and `append()` is that if the specified key already exists, `set()` will overwrite all existing values with the new one, whereas `append()` will append the new value onto the end of the existing set of values.
|
|
40
|
-
*
|
|
41
|
-
* @param name The name of the field whose data is contained in `value`.
|
|
42
|
-
* @param value The field's value. This can be [`Blob`](https://developer.mozilla.org/en-US/docs/Web/API/Blob)
|
|
43
|
-
or [`File`](https://developer.mozilla.org/en-US/docs/Web/API/File). If none of these are specified the value is converted to a string.
|
|
44
|
-
* @param fileName The filename reported to the server, when a Blob or File is passed as the second parameter. The default filename for Blob objects is "blob". The default filename for File objects is the file's filename.
|
|
45
|
-
*/
|
|
46
|
-
append(name: string, value: unknown, fileName?: string): void;
|
|
47
|
-
/**
|
|
48
|
-
* Returns all the values associated with a given key from within a `FormData` object.
|
|
49
|
-
*
|
|
50
|
-
* @param {string} name A name of the value you want to retrieve.
|
|
51
|
-
*
|
|
52
|
-
* @returns An array of `FormDataEntryValue` whose key matches the value passed in the `name` parameter. If the key doesn't exist, the method returns an empty list.
|
|
53
|
-
*/
|
|
54
|
-
getAll(name: string): FormDataEntryValue[];
|
|
55
|
-
/**
|
|
56
|
-
* Returns an [`iterator`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Iteration_protocols) allowing to go through the `FormData` key/value pairs.
|
|
57
|
-
* The key of each pair is a string; the value is a [`FormDataValue`](https://developer.mozilla.org/en-US/docs/Web/API/FormDataEntryValue).
|
|
58
|
-
*/
|
|
59
|
-
entries(): IterableIterator<[string, FormDataEntryValue]>;
|
|
60
|
-
/**
|
|
61
|
-
* An alias for FormDataLike#entries()
|
|
62
|
-
*/
|
|
63
|
-
[Symbol.iterator](): IterableIterator<[string, FormDataEntryValue]>;
|
|
64
|
-
readonly [Symbol.toStringTag]?: string;
|
|
65
|
-
}
|
|
66
|
-
export {};
|
|
67
|
-
//# sourceMappingURL=form-data-like.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"form-data-like.d.ts","sourceRoot":"","sources":["../../src/http_clients/form-data-like.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,UAAU,QAAQ;IACd;;OAEG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB;;OAEG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB;;OAEG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB;;OAEG;IACH,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B;;OAEG;IACH,MAAM,IAAI,cAAc,CAAC,UAAU,CAAC,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC;IACjE,QAAQ,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,EAAE,MAAM,CAAC;CAC1C;AAED;;GAEG;AACH,KAAK,kBAAkB,GAAG,MAAM,GAAG,QAAQ,CAAC;AAC5C;;GAEG;AACH,MAAM,WAAW,YAAY;IACzB;;;;;;;;;;OAUG;IACH,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9D;;;;;;OAMG;IACH,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,kBAAkB,EAAE,CAAC;IAC3C;;;OAGG;IACH,OAAO,IAAI,gBAAgB,CAAC,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC,CAAC;IAC1D;;OAEG;IACH,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,gBAAgB,CAAC,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC,CAAC;IACpE,QAAQ,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,EAAE,MAAM,CAAC;CAC1C"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"form-data-like.js","sourceRoot":"","sources":["../../src/http_clients/form-data-like.ts"],"names":[],"mappings":"AAAA;;GAEG"}
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
import type { BaseHttpClient, SendRequestOptions, StreamOptions } from '@crawlee/types';
|
|
2
|
-
/**
|
|
3
|
-
* A HTTP client implementation based on the `got-scraping` library.
|
|
4
|
-
*/
|
|
5
|
-
export declare class GotScrapingHttpClient implements BaseHttpClient {
|
|
6
|
-
/**
|
|
7
|
-
* Type guard that validates the HTTP method (excluding CONNECT).
|
|
8
|
-
* @param request - The HTTP request to validate
|
|
9
|
-
*/
|
|
10
|
-
private validateRequest;
|
|
11
|
-
private iterateHeaders;
|
|
12
|
-
private parseHeaders;
|
|
13
|
-
/**
|
|
14
|
-
* @inheritDoc
|
|
15
|
-
*/
|
|
16
|
-
sendRequest(request: Request, options?: SendRequestOptions): Promise<Response>;
|
|
17
|
-
/**
|
|
18
|
-
* @inheritDoc
|
|
19
|
-
*/
|
|
20
|
-
stream(request: Request, options?: StreamOptions): Promise<Response>;
|
|
21
|
-
}
|
|
22
|
-
//# sourceMappingURL=got-scraping-http-client.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"got-scraping-http-client.d.ts","sourceRoot":"","sources":["../../src/http_clients/got-scraping-http-client.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAMxF;;GAEG;AACH,qBAAa,qBAAsB,YAAW,cAAc;IACxD;;;OAGG;IACH,OAAO,CAAC,eAAe;IAMvB,OAAO,CAAE,cAAc;IAmBvB,OAAO,CAAC,YAAY;IAIpB;;OAEG;IACG,WAAW,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,QAAQ,CAAC;IA8BpF;;OAEG;IACG,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,QAAQ,CAAC;CA8D7E"}
|
|
@@ -1,116 +0,0 @@
|
|
|
1
|
-
import { Readable } from 'node:stream';
|
|
2
|
-
import { gotScraping } from 'got-scraping';
|
|
3
|
-
import { ResponseWithUrl } from './base-http-client.js';
|
|
4
|
-
/**
|
|
5
|
-
* A HTTP client implementation based on the `got-scraping` library.
|
|
6
|
-
*/
|
|
7
|
-
export class GotScrapingHttpClient {
|
|
8
|
-
/**
|
|
9
|
-
* Type guard that validates the HTTP method (excluding CONNECT).
|
|
10
|
-
* @param request - The HTTP request to validate
|
|
11
|
-
*/
|
|
12
|
-
validateRequest(request) {
|
|
13
|
-
return !['CONNECT', 'connect'].includes(request.method);
|
|
14
|
-
}
|
|
15
|
-
*iterateHeaders(headers) {
|
|
16
|
-
for (const [key, value] of Object.entries(headers)) {
|
|
17
|
-
// Filter out pseudo-headers
|
|
18
|
-
if (key.startsWith(':') || value === undefined) {
|
|
19
|
-
continue;
|
|
20
|
-
}
|
|
21
|
-
if (Array.isArray(value)) {
|
|
22
|
-
for (const v of value) {
|
|
23
|
-
yield [key, v];
|
|
24
|
-
}
|
|
25
|
-
}
|
|
26
|
-
else {
|
|
27
|
-
yield [key, value];
|
|
28
|
-
}
|
|
29
|
-
}
|
|
30
|
-
}
|
|
31
|
-
parseHeaders(headers) {
|
|
32
|
-
return new Headers([...this.iterateHeaders(headers)]);
|
|
33
|
-
}
|
|
34
|
-
/**
|
|
35
|
-
* @inheritDoc
|
|
36
|
-
*/
|
|
37
|
-
async sendRequest(request, options) {
|
|
38
|
-
const { session, timeout, proxyUrl } = options ?? {};
|
|
39
|
-
if (!this.validateRequest(request)) {
|
|
40
|
-
throw new Error(`The HTTP method CONNECT is not supported by the GotScrapingHttpClient.`);
|
|
41
|
-
}
|
|
42
|
-
const gotResult = await gotScraping({
|
|
43
|
-
url: request.url,
|
|
44
|
-
method: request.method,
|
|
45
|
-
headers: Object.fromEntries(request.headers.entries()),
|
|
46
|
-
body: request.body ? Readable.fromWeb(request.body) : undefined,
|
|
47
|
-
proxyUrl: proxyUrl ?? session?.proxyInfo?.url,
|
|
48
|
-
timeout: { request: timeout },
|
|
49
|
-
// We set cookieJar to undefined because
|
|
50
|
-
// `HttpCrawler` reads the cookies beforehand and sets them in `request.headers`.
|
|
51
|
-
// Using the `cookieJar` option directly would override that.
|
|
52
|
-
cookieJar: undefined,
|
|
53
|
-
});
|
|
54
|
-
const responseHeaders = this.parseHeaders(gotResult.headers);
|
|
55
|
-
return new ResponseWithUrl(new Uint8Array(gotResult.rawBody), {
|
|
56
|
-
headers: responseHeaders,
|
|
57
|
-
status: gotResult.statusCode,
|
|
58
|
-
statusText: gotResult.statusMessage ?? '',
|
|
59
|
-
url: gotResult.url,
|
|
60
|
-
});
|
|
61
|
-
}
|
|
62
|
-
/**
|
|
63
|
-
* @inheritDoc
|
|
64
|
-
*/
|
|
65
|
-
async stream(request, options) {
|
|
66
|
-
const { session, timeout } = options ?? {};
|
|
67
|
-
if (!this.validateRequest(request)) {
|
|
68
|
-
throw new Error(`The HTTP method CONNECT is not supported by the GotScrapingHttpClient.`);
|
|
69
|
-
}
|
|
70
|
-
// eslint-disable-next-line no-async-promise-executor
|
|
71
|
-
return new Promise(async (resolve, reject) => {
|
|
72
|
-
const stream = gotScraping({
|
|
73
|
-
url: request.url,
|
|
74
|
-
method: request.method,
|
|
75
|
-
headers: Object.fromEntries(request.headers.entries()),
|
|
76
|
-
body: request.body ? Readable.fromWeb(request.body) : undefined,
|
|
77
|
-
isStream: true,
|
|
78
|
-
proxyUrl: session?.proxyInfo?.url,
|
|
79
|
-
timeout: { request: timeout },
|
|
80
|
-
cookieJar: undefined,
|
|
81
|
-
});
|
|
82
|
-
stream.on('redirect', (updatedOptions, redirectResponse) => {
|
|
83
|
-
const nativeRedirectResponse = new ResponseWithUrl(redirectResponse.rawBody, {
|
|
84
|
-
headers: redirectResponse.headers,
|
|
85
|
-
status: redirectResponse.statusCode,
|
|
86
|
-
statusText: redirectResponse.statusMessage,
|
|
87
|
-
url: redirectResponse.url,
|
|
88
|
-
});
|
|
89
|
-
const nativeHeaders = new Headers(Object.entries(updatedOptions.headers)
|
|
90
|
-
.map(([key, value]) => (Array.isArray(value) ? value.map((v) => [key, v]) : [[key, value]]))
|
|
91
|
-
.flat());
|
|
92
|
-
options?.onRedirect?.(nativeRedirectResponse, {
|
|
93
|
-
url: updatedOptions.url,
|
|
94
|
-
headers: nativeHeaders,
|
|
95
|
-
});
|
|
96
|
-
updatedOptions.headers = Object.fromEntries(nativeHeaders.entries());
|
|
97
|
-
});
|
|
98
|
-
// We need to end the stream for DELETE requests, otherwise it will hang.
|
|
99
|
-
if (request.method && ['DELETE', 'delete'].includes(request.method)) {
|
|
100
|
-
stream.end();
|
|
101
|
-
}
|
|
102
|
-
stream.on('error', reject);
|
|
103
|
-
stream.on('response', (response) => {
|
|
104
|
-
const headers = this.parseHeaders(response.headers);
|
|
105
|
-
// Cast shouldn't be needed here, undici might have a different `ReadableStream` type
|
|
106
|
-
resolve(new ResponseWithUrl(Readable.toWeb(stream), {
|
|
107
|
-
status: response.statusCode,
|
|
108
|
-
statusText: response.statusMessage ?? '',
|
|
109
|
-
headers,
|
|
110
|
-
url: response.url,
|
|
111
|
-
}));
|
|
112
|
-
});
|
|
113
|
-
});
|
|
114
|
-
}
|
|
115
|
-
}
|
|
116
|
-
//# sourceMappingURL=got-scraping-http-client.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"got-scraping-http-client.js","sourceRoot":"","sources":["../../src/http_clients/got-scraping-http-client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAIvC,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAE3C,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD;;GAEG;AACH,MAAM,OAAO,qBAAqB;IAC9B;;;OAGG;IACK,eAAe,CACnB,OAAgB;QAEhB,OAAO,CAAC,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAO,CAAC,CAAC;IAC7D,CAAC;IAEO,CAAC,cAAc,CACnB,OAAsD;QAEtD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YACjD,4BAA4B;YAC5B,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC7C,SAAS;YACb,CAAC;YAED,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;gBACvB,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;oBACpB,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;gBACnB,CAAC;YACL,CAAC;iBAAM,CAAC;gBACJ,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YACvB,CAAC;QACL,CAAC;IACL,CAAC;IAEO,YAAY,CAAC,OAAsD;QACvE,OAAO,IAAI,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAC1D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CAAC,OAAgB,EAAE,OAA4B;QAC5D,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,OAAO,IAAI,EAAE,CAAC;QAErD,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CAAC,wEAAwE,CAAC,CAAC;QAC9F,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC;YAChC,GAAG,EAAE,OAAO,CAAC,GAAI;YACjB,MAAM,EAAE,OAAO,CAAC,MAA2B;YAC3C,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YACtD,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,IAAW,CAAC,CAAC,CAAC,CAAC,SAAS;YACtE,QAAQ,EAAE,QAAQ,IAAI,OAAO,EAAE,SAAS,EAAE,GAAG;YAC7C,OAAO,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE;YAC7B,wCAAwC;YACxC,iFAAiF;YACjF,6DAA6D;YAC7D,SAAS,EAAE,SAAS;SACvB,CAAC,CAAC;QAEH,MAAM,eAAe,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAE7D,OAAO,IAAI,eAAe,CAAC,IAAI,UAAU,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE;YAC1D,OAAO,EAAE,eAAe;YACxB,MAAM,EAAE,SAAS,CAAC,UAAU;YAC5B,UAAU,EAAE,SAAS,CAAC,aAAa,IAAI,EAAE;YACzC,GAAG,EAAE,SAAS,CAAC,GAAG;SACrB,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,OAAgB,EAAE,OAAuB;QAClD,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,OAAO,IAAI,EAAE,CAAC;QAE3C,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CAAC,wEAAwE,CAAC,CAAC;QAC9F,CAAC;QACD,qDAAqD;QACrD,OAAO,IAAI,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE;YACzC,MAAM,MAAM,GAAG,WAAW,CAAC;gBACvB,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,MAAM,EAAE,OAAO,CAAC,MAA2B;gBAC3C,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;gBACtD,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,IAAW,CAAC,CAAC,CAAC,CAAC,SAAS;gBACtE,QAAQ,EAAE,IAAI;gBACd,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG;gBACjC,OAAO,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE;gBAC7B,SAAS,EAAE,SAAS;aACvB,CAAC,CAAC;YAEH,MAAM,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,cAAuB,EAAE,gBAAqB,EAAE,EAAE;gBACrE,MAAM,sBAAsB,GAAG,IAAI,eAAe,CAAC,gBAAgB,CAAC,OAAO,EAAE;oBACzE,OAAO,EAAE,gBAAgB,CAAC,OAAO;oBACjC,MAAM,EAAE,gBAAgB,CAAC,UAAU;oBACnC,UAAU,EAAE,gBAAgB,CAAC,aAAa;oBAC1C,GAAG,EAAE,gBAAgB,CAAC,GAAG;iBAC5B,CAAC,CAAC;gBAEH,MAAM,aAAa,GAAG,IAAI,OAAO,CAC7B,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,OAAO,CAAC;qBACjC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;qBAC3F,IAAI,EAAwB,CACpC,CAAC;gBAEF,OAAO,EAAE,UAAU,EAAE,CAAC,sBAAsB,EAAE;oBAC1C,GAAG,EAAE,cAAc,CAAC,GAAG;oBACvB,OAAO,EAAE,aAAa;iBACzB,CAAC,CAAC;gBAEH,cAAc,CAAC,OAAO,GAAG,MAAM,CAAC,WAAW,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC;YACzE,CAAC,CAAC,CAAC;YAEH,yEAAyE;YACzE,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAClE,MAAM,CAAC,GAAG,EAAE,CAAC;YACjB,CAAC;YAED,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAE3B,MAAM,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,QAAuB,EAAE,EAAE;gBAC9C,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;gBACpD,qFAAqF;gBACrF,OAAO,CACH,IAAI,eAAe,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAQ,EAAE;oBAC/C,MAAM,EAAE,QAAQ,CAAC,UAAU;oBAC3B,UAAU,EAAE,QAAQ,CAAC,aAAa,IAAI,EAAE;oBACxC,OAAO;oBACP,GAAG,EAAE,QAAQ,CAAC,GAAG;iBACpB,CAAC,CACL,CAAC;YACN,CAAC,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;IACP,CAAC;CACJ"}
|
package/http_clients/index.d.ts
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/http_clients/index.ts"],"names":[],"mappings":"AAAA,cAAc,uBAAuB,CAAC;AACtC,cAAc,+BAA+B,CAAC"}
|
package/http_clients/index.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/http_clients/index.ts"],"names":[],"mappings":"AAAA,cAAc,uBAAuB,CAAC;AACtC,cAAc,+BAA+B,CAAC"}
|