@crawlee/http 4.0.0-beta.21 → 4.0.0-beta.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/internals/file-download.js +1 -1
- package/internals/file-download.js.map +1 -1
- package/internals/http-crawler.d.ts.map +1 -1
- package/internals/http-crawler.js +7 -20
- package/internals/http-crawler.js.map +1 -1
- package/internals/utils.d.ts +5 -0
- package/internals/utils.d.ts.map +1 -1
- package/internals/utils.js +36 -0
- package/internals/utils.js.map +1 -1
- package/package.json +6 -6
|
@@ -118,7 +118,7 @@ export class FileDownload extends BasicCrawler {
|
|
|
118
118
|
});
|
|
119
119
|
}
|
|
120
120
|
async initiateDownload(context) {
|
|
121
|
-
const response = await this.httpClient.
|
|
121
|
+
const response = await this.httpClient.sendRequest(context.request.intoFetchAPIRequest(), {
|
|
122
122
|
session: context.session,
|
|
123
123
|
});
|
|
124
124
|
const { type, charset: encoding } = parseContentTypeFromResponse(response);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"file-download.js","sourceRoot":"","sources":["../../src/internals/file-download.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAGhD,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAK/D,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAE,4BAA4B,EAAE,MAAM,YAAY,CAAC;AAsB1D;;;;;;;;GAQG;AACH,MAAM,UAAU,kBAAkB,CAAC,EAC/B,YAAY,EACZ,eAAe,GAAG,IAAI,EACtB,qBAAqB,EAAE,uBAAuB,GAAG,GAAG,GAKvD;IACG,IAAI,SAAS,GAA2C,EAAE,CAAC;IAE3D,MAAM,aAAa,GAAG,WAAW,CAAC,GAAG,EAAE;QACnC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEvB,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,GAAG,GAAG,QAAQ,CAAC,SAAS,GAAG,eAAe,CAAC,CAAC;QACvF,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,QAAQ,EAAE,EAAE,CAAC,GAAG,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAChF,MAAM,OAAO,GAAG,CAAC,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;QAE9D,IAAI,UAAU,GAAG,IAAI,GAAG,OAAO,GAAG,YAAY,EAAE,CAAC;YAC7C,aAAa,CAAC,aAAa,CAAC,CAAC;YAC7B,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC,CAAC;QAC1E,CAAC;IACL,CAAC,EAAE,uBAAuB,CAAC,CAAC;IAE5B,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;QACzB,SAAS,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE;YAC9B,SAAS,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;YAC/D,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC1B,CAAC;QACD,KAAK,EAAE,CAAC,QAAQ,EAAE,EAAE;YAChB,aAAa,CAAC,aAAa,CAAC,CAAC;YAC7B,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAClB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB,CAAC,EAC9B,mBAAmB,EACnB,eAAe,GAAG,IAAI,GAIzB;IACG,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,IAAI,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,OAAO,IAAI,SAAS,CAAC;QACjB,SAAS,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE;YAC9B,gBAAgB,IAAI,KAAK,CAAC,MAAM,CAAC;YAEjC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,GAAG,eAAe,EAAE,CAAC;gBAC7C,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACzB,mBAAmB,CAAC,gBAAgB,CAAC,CAAC;YAC1C,CAAC;YAED,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC1B,CAAC;QACD,KAAK,EAAE,CAAC,QAAQ,EAAE,EAAE;YAChB,mBAAmB,CAAC,gBAAgB,CAAC,CAAC;YACtC,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;AACP,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AACH,MAAM,OAAO,YAAa,SAAQ,YAAyC;IACvE,aAAa;IACb,YAAY,UAA4D,EAAE;QACtE,KAAK,CAAC;YACF,GAAG,OAAO;YACV,sBAAsB,EAAE,GAAG,EAAE,CACzB,eAAe,CAAC,MAAM,EAAmB,CAAC,OAAO,CAAC;gBAC9C,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC;gBACzD,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;oBACvB,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAW,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC/F,CAAC;aACJ,CAAC;SACT,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,OAAwB;QACnD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,
|
|
1
|
+
{"version":3,"file":"file-download.js","sourceRoot":"","sources":["../../src/internals/file-download.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAGhD,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAK/D,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAE,4BAA4B,EAAE,MAAM,YAAY,CAAC;AAsB1D;;;;;;;;GAQG;AACH,MAAM,UAAU,kBAAkB,CAAC,EAC/B,YAAY,EACZ,eAAe,GAAG,IAAI,EACtB,qBAAqB,EAAE,uBAAuB,GAAG,GAAG,GAKvD;IACG,IAAI,SAAS,GAA2C,EAAE,CAAC;IAE3D,MAAM,aAAa,GAAG,WAAW,CAAC,GAAG,EAAE;QACnC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEvB,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,GAAG,GAAG,QAAQ,CAAC,SAAS,GAAG,eAAe,CAAC,CAAC;QACvF,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,QAAQ,EAAE,EAAE,CAAC,GAAG,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAChF,MAAM,OAAO,GAAG,CAAC,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;QAE9D,IAAI,UAAU,GAAG,IAAI,GAAG,OAAO,GAAG,YAAY,EAAE,CAAC;YAC7C,aAAa,CAAC,aAAa,CAAC,CAAC;YAC7B,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC,CAAC;QAC1E,CAAC;IACL,CAAC,EAAE,uBAAuB,CAAC,CAAC;IAE5B,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;QACzB,SAAS,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE;YAC9B,SAAS,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;YAC/D,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC1B,CAAC;QACD,KAAK,EAAE,CAAC,QAAQ,EAAE,EAAE;YAChB,aAAa,CAAC,aAAa,CAAC,CAAC;YAC7B,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAClB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB,CAAC,EAC9B,mBAAmB,EACnB,eAAe,GAAG,IAAI,GAIzB;IACG,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,IAAI,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,OAAO,IAAI,SAAS,CAAC;QACjB,SAAS,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE;YAC9B,gBAAgB,IAAI,KAAK,CAAC,MAAM,CAAC;YAEjC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,GAAG,eAAe,EAAE,CAAC;gBAC7C,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACzB,mBAAmB,CAAC,gBAAgB,CAAC,CAAC;YAC1C,CAAC;YAED,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC1B,CAAC;QACD,KAAK,EAAE,CAAC,QAAQ,EAAE,EAAE;YAChB,mBAAmB,CAAC,gBAAgB,CAAC,CAAC;YACtC,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;AACP,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AACH,MAAM,OAAO,YAAa,SAAQ,YAAyC;IACvE,aAAa;IACb,YAAY,UAA4D,EAAE;QACtE,KAAK,CAAC;YACF,GAAG,OAAO;YACV,sBAAsB,EAAE,GAAG,EAAE,CACzB,eAAe,CAAC,MAAM,EAAmB,CAAC,OAAO,CAAC;gBAC9C,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC;gBACzD,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;oBACvB,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAW,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC/F,CAAC;aACJ,CAAC;SACT,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,OAAwB;QACnD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,mBAAmB,EAAE,EAAE;YACtF,OAAO,EAAE,OAAO,CAAC,OAAO;SAC3B,CAAC,CAAC;QAEH,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,4BAA4B,CAAC,QAAQ,CAAC,CAAC;QAE3E,OAAO,CAAC,OAAO,CAAC,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC;QAEnC,MAAM,gBAAgB,GAAG;YACrB,OAAO,EAAE,OAAO,CAAC,OAAiC;YAClD,QAAQ;YACR,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;SAClC,CAAC;QAEF,OAAO,gBAAgB,CAAC;IAC5B,CAAC;CACJ;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,UAAU,gBAAgB,CAG9B,MAAwC;IACtC,OAAO,MAAM,CAAC,MAAM,CAAU,MAAM,CAAC,CAAC;AAC1C,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/http-crawler.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAER,mBAAmB,EACnB,eAAe,EACf,YAAY,EACZ,sBAAsB,EACtB,OAAO,IAAI,cAAc,EACzB,cAAc,EACd,sBAAsB,EACtB,YAAY,EACZ,OAAO,EACV,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACH,YAAY,EAEZ,aAAa,EACb,eAAe,
|
|
1
|
+
{"version":3,"file":"http-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/http-crawler.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAER,mBAAmB,EACnB,eAAe,EACf,YAAY,EACZ,sBAAsB,EACtB,OAAO,IAAI,cAAc,EACzB,cAAc,EACd,sBAAsB,EACtB,YAAY,EACZ,OAAO,EACV,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACH,YAAY,EAEZ,aAAa,EACb,eAAe,EAKlB,MAAM,gBAAgB,CAAC;AACxB,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAEnD,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5D,OAAO,EAAE,KAAK,WAAW,EAAuB,MAAM,gBAAgB,CAAC;AAEvE,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAI9D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAsB3C,MAAM,MAAM,gBAAgB,CACxB,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,IAChC,YAAY,CAAC,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE1D,MAAM,WAAW,kBAAkB,CAC/B,OAAO,SAAS,2BAA2B,GAAG,2BAA2B,EACzE,gBAAgB,GAAG,UAAU,CAAC,KAAK,CAAC,EACpC,eAAe,SAAS,OAAO,GAAG,OAAO,GAAG,gBAAgB,CAC9D,SAAQ,mBAAmB,CAAC,OAAO,EAAE,gBAAgB,EAAE,eAAe,CAAC;IACrE;;OAEG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;OAEG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;;;;;;;;;;;;;;OAeG;IACH,kBAAkB,CAAC,EAAE,gBAAgB,CAAC,eAAe,CAAC,EAAE,CAAC;IAEzD;;;;;;;;;;;OAWG;IACH,mBAAmB,CAAC,EAAE,CAAC,CAAC,eAAe,EAAE,0BAA0B,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;IAE3F;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE/B;;;;;;;;;;OAUG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAEjC;;;;;;;;OAQG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;;;OAKG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAC;IAEnC;;;OAGG;IACH,0BAA0B,CAAC,EAAE,MAAM,EAAE,CAAC;IAEtC;;;OAGG;IACH,8BAA8B,CAAC,EAAE,MAAM,EAAE,CAAC;CAC7C;AAED;;GAEG;AACH,MAAM,MAAM,gBAAgB,CAAC,OAAO,IAAI,CAAC,eAAe,EAAE,OAAO,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAEtF,MAAM,MAAM,QAAQ,CAChB,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,IAChC,gBAAgB,CAAC,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE9D,UAAU,0BAA0B,CAChC,QAAQ,SAAS,UAAU,GAAG,GAAG,CACnC,SAAQ,eAAe,CAAC,QAAQ,CAAC;IAC/B;;OAEG;IACH,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,CAAC;IAEjD;;OAEG;IACH,QAAQ,EAAE,QAAQ,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,2BAA2B,CACxC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,CAClC,SAAQ,0BAA0B,CAAC,QAAQ,CAAC;IAC1C;;;;;OAKG;IACH,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;IAEtB;;OAEG;IACH,IAAI,EAAE,QAAQ,CAAC;IAEf;;OAEG;IACH,WAAW,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,cAAc,CAAA;KAAE,CAAC;IAExD;;;;;;;;;;;OAWG;IACH,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErE;;;;;;;;;;;OAWG;IACH,gBAAgB,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;CACjF;AAED,MAAM,WAAW,mBAAmB,CAAC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,QAAQ,SAAS,SAAS,GAAG,GAAG,CACpG,SAAQ,2BAA2B,CAAC,QAAQ,EAAE,QAAQ,CAAC;CAAG;AAE9D,MAAM,MAAM,kBAAkB,CAC1B,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,IAChC,cAAc,CAAC,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkEG;AACH,qBAAa,WAAW,CACpB,OAAO,SAAS,2BAA2B,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,2BAA2B,EACnF,gBAAgB,GAAG,UAAU,CAAC,KAAK,CAAC,EACpC,eAAe,SAAS,OAAO,GAAG,OAAO,GAAG,gBAAgB,CAC9D,SAAQ,YAAY,CAAC,OAAO,EAAE,gBAAgB,EAAE,eAAe,CAAC;aAmCxC,MAAM;IAlC5B,SAAS,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,eAAe,CAAC,EAAE,CAAC;IAClE,SAAS,CAAC,mBAAmB,EAAE,CAAC,CAAC,eAAe,EAAE,0BAA0B,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;IACpG,SAAS,CAAC,wBAAwB,EAAE,OAAO,CAAC;IAC5C,SAAS,CAAC,uBAAuB,EAAE,MAAM,CAAC;IAC1C,SAAS,CAAC,eAAe,EAAE,OAAO,CAAC;IACnC,SAAS,CAAC,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAC3C,SAAS,CAAC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IACzC,SAAS,CAAC,8BAA8B,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACtD,SAAS,CAAC,0BAA0B,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAClD,SAAS,CAAC,QAAQ,CAAC,kBAAkB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAEnD,iBAA0B,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAepC;IAEF;;OAEG;gBAEC,OAAO,GAAE,kBAAkB,CAAC,OAAO,EAAE,gBAAgB,EAAE,eAAe,CAAC,GACnE,sBAAsB,CAAC,2BAA2B,EAAE,OAAO,CAAa,EAC1D,MAAM,gBAAkC;IAkE9D,SAAS,CAAC,oBAAoB,IAAI,eAAe,CAAC,eAAe,EAAE,2BAA2B,CAAC;YASjF,eAAe;YAkDf,mBAAmB;YAuEnB,6BAA6B;cAQ3B,gBAAgB,CAAC,eAAe,EAAE,2BAA2B,GAAG,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC;IAyBvG;;;OAGG;IACH,SAAS,CAAC,aAAa,CACnB,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,eAAe,EACrC,cAAc,EAAE,MAAM,EACtB,eAAe,EAAE,MAAM,GACxB,MAAM;IAQT;;;;OAIG;cACa,gBAAgB,CAAC,EAC7B,OAAO,EACP,OAAO,EACP,QAAQ,EACR,YAAY,GACf,EAAE,sBAAsB,GAAG,OAAO,CAAC,QAAQ,CAAC;IAmB7C;;OAEG;cACa,cAAc,CAAC,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,QAAQ;;;;;;;;;;;;;;;IA2C1E;;OAEG;IACH,SAAS,CAAC,kBAAkB,CAAC,OAAO,EAAE,cAAc,EAAE,OAAO,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,MAAM;;;;;;;;;;;cAY/D,MAAM,GAAG,SAAS;;IAmB7C,SAAS,CAAC,eAAe,CACrB,OAAO,EAAE,cAAc,EACvB,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,cAAc,GACzB;QACC,QAAQ,EAAE,cAAc,CAAC;QACzB,QAAQ,EAAE,QAAQ,CAAC;KACtB;IAsCD;;OAEG;IACH,SAAS,CAAC,yBAAyB,CAAC,mBAAmB,EAAE,CAAC,MAAM,GAAG,WAAW,GAAG,YAAY,CAAC,EAAE;IAgBhG;;OAEG;IACH,SAAS,CAAC,qBAAqB,CAAC,OAAO,CAAC,EAAE,OAAO;IAKjD,OAAO,CAAC,oBAAoB;IAkB5B;;OAEG;IACH,OAAO,CAAC,iBAAiB,CA4BvB;CACL;AAED,UAAU,sBAAsB;IAC5B,OAAO,EAAE,cAAc,CAAC;IACxB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,gBAAgB,CAC5B,OAAO,SAAS,mBAAmB,GAAG,mBAAmB,EACzD,QAAQ,SAAS,UAAU,GAAG,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAC1E,MAAM,CAAC,EAAE,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,mDAEzC"}
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
import { Readable } from 'node:stream';
|
|
2
2
|
import util from 'node:util';
|
|
3
|
-
import { BasicCrawler, BLOCKED_STATUS_CODES, Configuration, ContextPipeline, mergeCookies,
|
|
3
|
+
import { BasicCrawler, BLOCKED_STATUS_CODES, Configuration, ContextPipeline, mergeCookies, RequestState, Router, SessionError, } from '@crawlee/basic';
|
|
4
|
+
import { ResponseWithUrl } from '@crawlee/http-client';
|
|
4
5
|
import { RETRY_CSS_SELECTORS } from '@crawlee/utils';
|
|
5
6
|
import * as cheerio from 'cheerio';
|
|
6
7
|
import contentTypeParser from 'content-type';
|
|
7
8
|
import iconv from 'iconv-lite';
|
|
8
9
|
import ow from 'ow';
|
|
9
10
|
import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
|
|
10
|
-
import { parseContentTypeFromResponse } from './utils.js';
|
|
11
|
-
let TimeoutError;
|
|
11
|
+
import { parseContentTypeFromResponse, processHttpRequestOptions } from './utils.js';
|
|
12
12
|
/**
|
|
13
13
|
* Default mime types, which HttpScraper supports.
|
|
14
14
|
*/
|
|
@@ -299,16 +299,12 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
299
299
|
* received content type matches text/html, application/xml, application/xhtml+xml.
|
|
300
300
|
*/
|
|
301
301
|
async _requestFunction({ request, session, proxyUrl, cookieString, }) {
|
|
302
|
-
if (!TimeoutError) {
|
|
303
|
-
// @ts-ignore
|
|
304
|
-
({ TimeoutError } = await import('got-scraping'));
|
|
305
|
-
}
|
|
306
302
|
const opts = this._getRequestOptions(request, session, proxyUrl);
|
|
307
303
|
try {
|
|
308
304
|
return await this._requestAsBrowser(opts, session, cookieString);
|
|
309
305
|
}
|
|
310
306
|
catch (e) {
|
|
311
|
-
if (e instanceof TimeoutError) {
|
|
307
|
+
if (e instanceof Error && e.constructor.name === 'TimeoutError') {
|
|
312
308
|
this._handleRequestTimeout(session);
|
|
313
309
|
return new Response(); // this will never happen, as _handleRequestTimeout always throws
|
|
314
310
|
}
|
|
@@ -351,10 +347,10 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
351
347
|
throw new Error(`${status} - Internal Server Error: ${body.slice(0, 100)}`);
|
|
352
348
|
}
|
|
353
349
|
else if (HTML_AND_XML_MIME_TYPES.includes(type)) {
|
|
354
|
-
return { response, contentType, body: await
|
|
350
|
+
return { response, contentType, body: await reencodedResponse.text() };
|
|
355
351
|
}
|
|
356
352
|
else {
|
|
357
|
-
const body = Buffer.from(await
|
|
353
|
+
const body = Buffer.from(await reencodedResponse.bytes());
|
|
358
354
|
return {
|
|
359
355
|
body,
|
|
360
356
|
response,
|
|
@@ -475,7 +471,7 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
475
471
|
opts.headers?.delete('cookie');
|
|
476
472
|
opts.headers?.set('Cookie', cookieString);
|
|
477
473
|
}
|
|
478
|
-
const response = await this.httpClient.
|
|
474
|
+
const response = await this.httpClient.sendRequest(new Request(opts.url, {
|
|
479
475
|
body: opts.body ? Readable.toWeb(opts.body) : undefined,
|
|
480
476
|
headers: new Headers(opts.headers),
|
|
481
477
|
method: opts.method,
|
|
@@ -484,15 +480,6 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
484
480
|
}), {
|
|
485
481
|
session,
|
|
486
482
|
timeout: opts.timeout,
|
|
487
|
-
onRedirect: (redirectResponse, updatedRequest) => {
|
|
488
|
-
if (this.persistCookiesPerSession) {
|
|
489
|
-
session.setCookiesFromResponse(redirectResponse);
|
|
490
|
-
const cookieStringRedirected = session.getCookieString(updatedRequest.url.toString());
|
|
491
|
-
if (cookieStringRedirected !== '') {
|
|
492
|
-
updatedRequest.headers.set('Cookie', cookieStringRedirected);
|
|
493
|
-
}
|
|
494
|
-
}
|
|
495
|
-
},
|
|
496
483
|
});
|
|
497
484
|
return response;
|
|
498
485
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http-crawler.js","sourceRoot":"","sources":["../../src/internals/http-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,IAAI,MAAM,WAAW,CAAC;AAc7B,OAAO,EACH,YAAY,EACZ,oBAAoB,EACpB,aAAa,EACb,eAAe,EACf,YAAY,EACZ,yBAAyB,EACzB,YAAY,EACZ,eAAe,EACf,MAAM,EACN,YAAY,GACf,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EAAoB,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AACvE,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,OAAO,iBAAiB,MAAM,cAAc,CAAC;AAE7C,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,EAAE,MAAM,IAAI,CAAC;AAGpB,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAEhE,OAAO,EAAE,4BAA4B,EAAE,MAAM,YAAY,CAAC;AAE1D,IAAI,YAAsC,CAAC;AAE3C;;GAEG;AACH,MAAM,uBAAuB,GAAG,CAAC,WAAW,EAAE,UAAU,EAAE,uBAAuB,EAAE,iBAAiB,CAAC,CAAC;AACtG,MAAM,0BAA0B,GAAG,kBAAkB,CAAC;AACtD,MAAM,sCAAsC,GAA0B;IAClE,kBAAkB,EAAE,EAAE;IACtB,kBAAkB,EAAE;QAChB,6BAA6B,EAAE,CAAC;QAChC,gBAAgB,EAAE,GAAG;KACxB;IACD,mBAAmB,EAAE;QACjB,2BAA2B,EAAE,GAAG;KACnC;CACJ,CAAC;AA+LF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkEG;AACH,MAAM,OAAO,WAIX,SAAQ,YAAwD;IAmCxC;IAlCZ,kBAAkB,CAAsC;IACxD,mBAAmB,CAAuE;IAC1F,wBAAwB,CAAU;IAClC,uBAAuB,CAAS;IAChC,eAAe,CAAU;IACzB,uBAAuB,CAAU;IACjC,qBAAqB,CAAU;IAC/B,8BAA8B,CAAc;IAC5C,0BAA0B,CAAc;IAC/B,kBAAkB,CAAc;IAEzC,MAAM,CAAU,YAAY,GAAG;QACrC,GAAG,YAAY,CAAC,YAAY;QAE5B,qBAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QACpC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;QACxD,uBAAuB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC3C,qBAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,wBAAwB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QAE7C,8BAA8B,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;QACnE,0BAA0B,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;QAE/D,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK;QACrC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK;KACzC,CAAC;IAEF;;OAEG;IACH,YACI,UACmE,EAAS,EAC1D,SAAS,aAAa,CAAC,eAAe,EAAE;QAE1D,EAAE,CAAC,OAAO,EAAE,oBAAoB,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC;QAElF,MAAM,EACF,qBAAqB,GAAG,EAAE,EAC1B,eAAe,GAAG,IAAI,EACtB,mBAAmB,GAAG,EAAE,EACxB,uBAAuB,EACvB,qBAAqB,EACrB,wBAAwB,EACxB,kBAAkB,GAAG,EAAE,EACvB,mBAAmB,GAAG,EAAE,EACxB,8BAA8B,GAAG,EAAE,EACnC,0BAA0B,GAAG,EAAE;QAE/B,eAAe;QACf,qBAAqB,GAAG,sCAAsC,EAC9D,sBAAsB,EACtB,GAAG,mBAAmB,EACzB,GAAG,OAAO,CAAC;QAEZ,KAAK,CACD;YACI,GAAG,mBAAmB;YACtB,qBAAqB;YACrB,sBAAsB,EAClB,sBAAsB;gBACtB,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,oBAAoB,EAA+C,CAAC;SACvF,EACD,MAAM,CACT,CAAC;QA/BgB,WAAM,GAAN,MAAM,CAAkC;QAiC1D,uEAAuE;QACvE,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,wBAAwB,EAAE,CAAC;YACnD,MAAM,IAAI,KAAK,CAAC,iFAAiF,CAAC,CAAC;QACvG,CAAC;QAED,IAAI,CAAC,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,uBAAuB,EAAE,0BAA0B,CAAC,CAAC,CAAC;QAC5F,IAAI,mBAAmB,CAAC,MAAM;YAAE,IAAI,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC;QAEpF,IAAI,uBAAuB,IAAI,qBAAqB,EAAE,CAAC;YACnD,IAAI,CAAC,GAAG,CAAC,OAAO,CACZ,sGAAsG,CACzG,CAAC;QACN,CAAC;QAED,IAAI,CAAC,uBAAuB,GAAG,qBAAqB,GAAG,IAAI,CAAC;QAC5D,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,uBAAuB,GAAG,uBAAuB,CAAC;QACvD,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,CAAC;QACnD,IAAI,CAAC,8BAA8B,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,8BAA8B,CAAC,CAAC,CAAC;QACnF,IAAI,CAAC,0BAA0B,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,0BAA0B,CAAC,CAAC,CAAC;QAC3E,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,mBAAmB,GAAG;YACvB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,oBAAoB,CAAC,OAAO,EAAE,QAAS,CAAC;YACxE,GAAG,mBAAmB;SACzB,CAAC;QAEF,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,CAAC,wBAAwB,GAAG,wBAAwB,IAAI,IAAI,CAAC;QACrE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,wBAAwB,GAAG,KAAK,CAAC;QAC1C,CAAC;IACL,CAAC;IAES,oBAAoB;QAC1B,OAAO,eAAe,CAAC,MAAM,EAAmB;aAC3C,OAAO,CAAC;YACL,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC;SAC1C,CAAC;aACD,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;aACxD,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,6BAA6B,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC5E,CAAC;IAEO,KAAK,CAAC,eAAe,CACzB,eAAgC;QAEhC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAE7C,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YACzB,OAAO;gBACH,OAAO,EAAE,IAAI,KAAK,CAAC,OAAO,EAAE;oBACxB,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ;wBAC9B,IAAI,YAAY,KAAK,WAAW,EAAE,CAAC;4BAC/B,MAAM,IAAI,KAAK,CACX,+EAA+E,CAClF,CAAC;wBACN,CAAC;wBACD,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ,CAAC,CAAC;oBACvD,CAAC;iBACJ,CAAkC;gBACnC,IAAI,QAAQ;oBACR,MAAM,IAAI,KAAK,CAAC,sEAAsE,CAAC,CAAC;gBAC5F,CAAC;aACJ,CAAC;QACN,CAAC;QAED,MAAM,yBAAyB,GAAG,IAAI,CAAC,2BAA2B,CAAC,OAAO,CAAC,CAAC;QAE5E,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,UAAU,CAAC;QACxC,qEAAqE;QACrE,8CAA8C;QAC9C,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,kBAAkB,EAAE,eAAe,CAAC,CAAC;QACnE,SAAS,EAAE,CAAC;QAEZ,MAAM,0BAA0B,GAAG,IAAI,CAAC,2BAA2B,CAAC,OAAO,CAAC,CAAC;QAE7E,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,CAAC,eAAe,EAAE,yBAAyB,EAAE,0BAA0B,CAAC,CAAC;QAEhH,MAAM,QAAQ,GAAG,eAAe,CAAC,SAAS,EAAE,GAAG,CAAC;QAEhD,MAAM,YAAY,GAAG,MAAM,mBAAmB,CAC1C,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC,EAC/E,IAAI,CAAC,uBAAuB,EAC5B,2BAA2B,IAAI,CAAC,uBAAuB,GAAG,IAAI,WAAW,CAC5E,CAAC;QACF,SAAS,EAAE,CAAC;QAEZ,OAAO,CAAC,SAAS,GAAG,YAAY,EAAE,GAAG,CAAC;QACtC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,SAAS,CAAC;QAEvC,OAAO,EAAE,OAAO,EAAE,OAAwC,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC;IACzF,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAC7B,eAA2C;QAI3C,IAAI,eAAe,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YACzC,OAAO;gBACH,IAAI,WAAW;oBACX,MAAM,IAAI,KAAK,CAAC,yEAAyE,CAAC,CAAC;gBAC/F,CAAC;gBACD,IAAI,IAAI;oBACJ,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;gBACxF,CAAC;gBACD,IAAI,IAAI;oBACJ,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;gBACxF,CAAC;gBACD,IAAI,eAAe;oBACf,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAC;gBACjG,CAAC;gBACD,IAAI,gBAAgB;oBAChB,MAAM,IAAI,KAAK,CAAC,4EAA4E,CAAC,CAAC;gBAClG,CAAC;aACJ,CAAC;QACN,CAAC;QAED,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,mBAAmB,EAAE,eAAe,CAAC,CAAC;QACpE,SAAS,EAAE,CAAC;QAEZ,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,OAAO,EAAE,eAAe,CAAC,QAAQ,CAAC,CAAC;QAC5F,SAAS,EAAE,CAAC;QACZ,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAS,CAAC;QAClC,MAAM,WAAW,GAAG,MAAM,CAAC,WAAY,CAAC;QAExC,MAAM,eAAe,GAAG,KAAK,EAAE,QAAgB,EAAE,UAAmB,EAAE,EAAE;YACpE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;YAEhD,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACjC,MAAM,IAAI,KAAK,CAAC,aAAa,QAAQ,cAAc,CAAC,CAAC;YACzD,CAAC;QACL,CAAC,CAAC;QACF,MAAM,gBAAgB,GAAG,KAAK,EAAE,QAAiB,EAAE,SAAkB,EAAE,EAAE;YACrE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;YAEhD,IAAI,QAAQ,EAAE,CAAC;gBACX,MAAO,eAA+C,CAAC,eAAe,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;YAChG,CAAC;YAED,OAAO,CAAC,CAAC;QACb,CAAC,CAAC;QAEF,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,CAAC,sBAAsB,CAAC,eAAe,CAAC,OAAQ,EAAE,QAAQ,CAAC,MAAO,CAAC,CAAC;QAC5E,CAAC;QAED,IAAI,IAAI,CAAC,wBAAwB,EAAE,CAAC;YAChC,eAAe,CAAC,OAAQ,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC;QAC9D,CAAC;QAED,OAAO;YACH,IAAI,IAAI;gBACJ,IAAI,WAAW,CAAC,IAAI,KAAK,0BAA0B;oBAAE,OAAO,IAAI,CAAC;gBACjE,MAAM,UAAU,GAAG,MAAM,CAAC,IAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;gBAC/D,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAClC,CAAC;YACD,eAAe;YACf,gBAAgB;YAChB,WAAW;YACX,IAAI,EAAE,MAAM,CAAC,IAAI;SACpB,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,6BAA6B,CAAC,eAA4C;QACpF,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC;YAC3D,IAAI,KAAK;gBAAE,MAAM,IAAI,YAAY,CAAC,KAAK,CAAC,CAAC;QAC7C,CAAC;QACD,OAAO,EAAE,CAAC;IACd,CAAC;IAES,KAAK,CAAC,gBAAgB,CAAC,eAA4C;QACzE,IAAI,uBAAuB,CAAC,QAAQ,CAAC,eAAe,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;YACrE,MAAM,CAAC,GAAG,MAAM,eAAe,CAAC,gBAAgB,EAAE,CAAC;YAEnD,MAAM,cAAc,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAExF,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,OAAO,oBAAoB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3D,CAAC;QACL,CAAC;QAED,MAAM,kBAAkB;QACpB,wCAAwC;QACxC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,oBAAoB,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC;YACtD,CAAC,CAAC,wCAAwC;gBACxC,IAAI,CAAC,WAAY,CAAC,oBAAoB,CAAC;YACzC,CAAC,CAAC,oBAAoB,CAAC;QAE/B,IAAI,kBAAkB,CAAC,QAAQ,CAAC,eAAe,CAAC,QAAQ,CAAC,MAAO,CAAC,EAAE,CAAC;YAChE,OAAO,0BAA0B,eAAe,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;QACvE,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;OAGG;IACO,aAAa,CACnB,EAAE,OAAO,EAAE,OAAO,EAAmB,EACrC,cAAsB,EACtB,eAAuB;QAEvB,MAAM,aAAa,GAAG,OAAO,EAAE,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QAElE,MAAM,aAAa,GAAG,CAAC,aAAa,EAAE,cAAc,EAAE,eAAe,CAAC,CAAC;QAEvE,OAAO,YAAY,CAAC,OAAO,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;IACpD,CAAC;IAED;;;;OAIG;IACO,KAAK,CAAC,gBAAgB,CAAC,EAC7B,OAAO,EACP,OAAO,EACP,QAAQ,EACR,YAAY,GACS;QACrB,IAAI,CAAC,YAAY,EAAE,CAAC;YAChB,aAAa;YACb,CAAC,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC;QACtD,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;QAEjE,IAAI,CAAC;YACD,OAAO,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC;QACrE,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,YAAY,YAAY,EAAE,CAAC;gBAC5B,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;gBACpC,OAAO,IAAI,QAAQ,EAAE,CAAC,CAAC,iEAAiE;YAC5F,CAAC;YAED,IAAI,IAAI,CAAC,YAAY,CAAC,CAAU,CAAC,EAAE,CAAC;gBAChC,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAU,CAAW,CAAC,CAAC;YAC5E,CAAC;iBAAM,CAAC;gBACJ,MAAM,CAAC,CAAC;YACZ,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,cAAc,CAAC,OAAuB,EAAE,QAAkB;QACtE,MAAM,EAAE,MAAM,EAAE,GAAG,QAAQ,CAAC;QAC5B,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,4BAA4B,CAAC,QAAQ,CAAC,CAAC;QACjE,MAAM,EAAE,QAAQ,EAAE,iBAAiB,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;QACnG,MAAM,WAAW,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QAEvC,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;YACjC,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;QAC1C,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,CAAC,0BAA0B,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACjE,MAAM,YAAY,GAAG,IAAI,CAAC,8BAA8B,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAErE,IAAI,CAAC,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,YAAY,EAAE,CAAC;YACnD,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,IAAI,EAAE,CAAC,CAAC,oGAAoG;YAEjJ,2DAA2D;YAC3D,gDAAgD;YAChD,IAAI,IAAI,KAAK,0BAA0B,EAAE,CAAC;gBACtC,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACvC,IAAI,EAAE,OAAO,EAAE,GAAG,aAAa,CAAC;gBAChC,IAAI,CAAC,OAAO;oBAAE,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,cAAc,EAAE,EAAE,EAAE,CAAC,CAAC;gBACtF,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,MAAM,OAAO,EAAE,CAAC,CAAC;YAC9C,CAAC;YAED,IAAI,YAAY,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,uCAAuC,CAAC,CAAC;YACtE,CAAC;YAED,8EAA8E;YAC9E,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,6BAA6B,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAChF,CAAC;aAAM,IAAI,uBAAuB,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAChD,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;QAClE,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;YACjD,OAAO;gBACH,IAAI;gBACJ,QAAQ;gBACR,WAAW;aACd,CAAC;QACN,CAAC;IACL,CAAC;IAED;;OAEG;IACO,kBAAkB,CAAC,OAAuB,EAAE,OAAiB,EAAE,QAAiB;QACtF,MAAM,cAAc,GAAG;YACnB,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,QAAQ;YACR,OAAO,EAAE,IAAI,CAAC,uBAAuB;YACrC,SAAS,EAAE,IAAI,CAAC,wBAAwB,CAAC,CAAC,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,SAAS;YACzE,YAAY,EAAE,OAAO;YACrB,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,KAAK,EAAE;gBACH,kBAAkB,EAAE,CAAC,IAAI,CAAC,eAAe;aAC5C;YACD,IAAI,EAAE,SAA+B;SACxC,CAAC;QAEF,yHAAyH;QACzH,OAAO,CAAC,cAAc,CAAC,cAAc,CAAC,OAAQ,EAAE,QAAQ,CAAC,CAAC;QAE1D,4CAA4C;QAC5C,IAAI,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,CAAC;YACtC,cAAc,CAAC,KAAK,GAAG;gBACnB,GAAG,cAAc,CAAC,KAAK;gBACvB,kBAAkB,EAAE,KAAK;aAC5B,CAAC;QACN,CAAC;QAED,IAAI,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,cAAc,CAAC,IAAI,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;QAEvF,OAAO,cAAc,CAAC;IAC1B,CAAC;IAES,eAAe,CACrB,OAAuB,EACvB,QAAkB,EAClB,QAAwB;QAKxB,IAAI,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAC7B,QAAQ,GAAG,IAAI,CAAC,qBAAuC,CAAC;QAC5D,CAAC;aAAM,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,uBAAuB,EAAE,CAAC;YACnD,QAAQ,GAAG,IAAI,CAAC,uBAAyC,CAAC;QAC9D,CAAC;QAED,sDAAsD;QACtD,MAAM,IAAI,GAAG,MAAM,CAAC;QACpB,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAEnD,2DAA2D;QAC3D,+CAA+C;QAC/C,IAAI,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;QAE/D,+DAA+D;QAC/D,IAAI,KAAK,CAAC,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC;YACjC,MAAM,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YACxG,MAAM,aAAa,GAAG,QAAQ,CAAC,IAAI;gBAC/B,CAAC,CAAC,QAAQ,CAAC,KAAK,CACV,QAAQ,CAAC,IAAI,CACT,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAW,CAAC;qBACjC,IAAI,CAAC,YAAY,CAAC;qBAClB,IAAI,CAAC,YAAY,CAAC,CAC1B,CACJ;gBACH,CAAC,CAAC,IAAI,CAAC;YAEX,OAAO;gBACH,QAAQ,EAAE,IAAI,eAAe,CAAC,aAAoB,EAAE,QAAQ,CAAC;gBAC7D,QAAQ,EAAE,IAAI;aACjB,CAAC;QACN,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,YAAY,OAAO,CAAC,GAAG,8CAA8C,QAAQ,EAAE,CAAC,CAAC;IACrG,CAAC;IAED;;OAEG;IACO,yBAAyB,CAAC,mBAA4D;QAC5F,KAAK,MAAM,QAAQ,IAAI,mBAAmB,EAAE,CAAC;YACzC,IAAI,QAAQ,KAAK,KAAK,EAAE,CAAC;gBACrB,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBACtC,SAAS;YACb,CAAC;YAED,IAAI,CAAC;gBACD,MAAM,UAAU,GAAG,iBAAiB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBACrD,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;YACjD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,IAAI,KAAK,CAAC,2BAA2B,QAAQ,sCAAsC,CAAC,CAAC;YAC/F,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACO,qBAAqB,CAAC,OAAiB;QAC7C,OAAO,EAAE,OAAO,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,uBAAuB,GAAG,IAAI,WAAW,CAAC,CAAC;IAC/F,CAAC;IAEO,oBAAoB,CAAC,OAAuB,EAAE,QAAkB;QACpE,MAAM,EAAE,MAAM,EAAE,GAAG,QAAQ,CAAC;QAC5B,MAAM,EAAE,IAAI,EAAE,GAAG,4BAA4B,CAAC,QAAQ,CAAC,CAAC;QAExD,sEAAsE;QACtE,MAAM,kBAAkB,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1F,wDAAwD;QACxD,MAAM,sBAAsB,GAAG,MAAM,IAAI,GAAG,IAAI,kBAAkB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAEpF,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;YACvG,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;YACvB,MAAM,IAAI,KAAK,CACX,YAAY,OAAO,CAAC,GAAG,wBAAwB,IAAI,IAAI;gBACnD,YAAY,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,kCAAkC,CACnG,CAAC;QACN,CAAC;IACL,CAAC;IAED;;OAEG;IACK,iBAAiB,GAAG,KAAK,EAAE,OAAwB,EAAE,OAAiB,EAAE,YAAqB,EAAE,EAAE;QACrG,MAAM,IAAI,GAAG,yBAAyB,CAAC;YACnC,GAAI,OAAe;YACnB,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,YAAY,EAAE,MAAM;SACvB,CAAC,CAAC;QAEH,IAAI,YAAY,EAAE,CAAC;YACf,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC/B,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC/B,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QAC9C,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CACzC,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE;YAClB,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAS,CAAC,CAAC,CAAC,SAAS;YAChE,OAAO,EAAE,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;YAClC,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,kEAAkE;YAClE,MAAM,EAAE,MAAM;SACF,CAAC,EACjB;YACI,OAAO;YACP,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,UAAU,EAAE,CAAC,gBAAgB,EAAE,cAAc,EAAE,EAAE;gBAC7C,IAAI,IAAI,CAAC,wBAAwB,EAAE,CAAC;oBAChC,OAAQ,CAAC,sBAAsB,CAAC,gBAAgB,CAAC,CAAC;oBAElD,MAAM,sBAAsB,GAAG,OAAQ,CAAC,eAAe,CAAC,cAAc,CAAC,GAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;oBACxF,IAAI,sBAAsB,KAAK,EAAE,EAAE,CAAC;wBAChC,cAAc,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,sBAAsB,CAAC,CAAC;oBACjE,CAAC;gBACL,CAAC;YACL,CAAC;SACJ,CACJ,CAAC;QAEF,OAAO,QAAQ,CAAC;IACpB,CAAC,CAAC;;AAUN;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,UAAU,gBAAgB,CAG9B,MAAwC;IACtC,OAAO,MAAM,CAAC,MAAM,CAAU,MAAM,CAAC,CAAC;AAC1C,CAAC"}
|
|
1
|
+
{"version":3,"file":"http-crawler.js","sourceRoot":"","sources":["../../src/internals/http-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,IAAI,MAAM,WAAW,CAAC;AAc7B,OAAO,EACH,YAAY,EACZ,oBAAoB,EACpB,aAAa,EACb,eAAe,EACf,YAAY,EACZ,YAAY,EACZ,MAAM,EACN,YAAY,GACf,MAAM,gBAAgB,CAAC;AAExB,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAEvD,OAAO,EAAoB,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AACvE,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,OAAO,iBAAiB,MAAM,cAAc,CAAC;AAC7C,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,EAAE,MAAM,IAAI,CAAC;AAGpB,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAEhE,OAAO,EAAE,4BAA4B,EAAE,yBAAyB,EAAE,MAAM,YAAY,CAAC;AAErF;;GAEG;AACH,MAAM,uBAAuB,GAAG,CAAC,WAAW,EAAE,UAAU,EAAE,uBAAuB,EAAE,iBAAiB,CAAC,CAAC;AACtG,MAAM,0BAA0B,GAAG,kBAAkB,CAAC;AACtD,MAAM,sCAAsC,GAA0B;IAClE,kBAAkB,EAAE,EAAE;IACtB,kBAAkB,EAAE;QAChB,6BAA6B,EAAE,CAAC;QAChC,gBAAgB,EAAE,GAAG;KACxB;IACD,mBAAmB,EAAE;QACjB,2BAA2B,EAAE,GAAG;KACnC;CACJ,CAAC;AA+LF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkEG;AACH,MAAM,OAAO,WAIX,SAAQ,YAAwD;IAmCxC;IAlCZ,kBAAkB,CAAsC;IACxD,mBAAmB,CAAuE;IAC1F,wBAAwB,CAAU;IAClC,uBAAuB,CAAS;IAChC,eAAe,CAAU;IACzB,uBAAuB,CAAU;IACjC,qBAAqB,CAAU;IAC/B,8BAA8B,CAAc;IAC5C,0BAA0B,CAAc;IAC/B,kBAAkB,CAAc;IAEzC,MAAM,CAAU,YAAY,GAAG;QACrC,GAAG,YAAY,CAAC,YAAY;QAE5B,qBAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QACpC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;QACxD,uBAAuB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC3C,qBAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,wBAAwB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QAE7C,8BAA8B,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;QACnE,0BAA0B,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;QAE/D,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK;QACrC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK;KACzC,CAAC;IAEF;;OAEG;IACH,YACI,UACmE,EAAS,EAC1D,SAAS,aAAa,CAAC,eAAe,EAAE;QAE1D,EAAE,CAAC,OAAO,EAAE,oBAAoB,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC;QAElF,MAAM,EACF,qBAAqB,GAAG,EAAE,EAC1B,eAAe,GAAG,IAAI,EACtB,mBAAmB,GAAG,EAAE,EACxB,uBAAuB,EACvB,qBAAqB,EACrB,wBAAwB,EACxB,kBAAkB,GAAG,EAAE,EACvB,mBAAmB,GAAG,EAAE,EACxB,8BAA8B,GAAG,EAAE,EACnC,0BAA0B,GAAG,EAAE;QAE/B,eAAe;QACf,qBAAqB,GAAG,sCAAsC,EAC9D,sBAAsB,EACtB,GAAG,mBAAmB,EACzB,GAAG,OAAO,CAAC;QAEZ,KAAK,CACD;YACI,GAAG,mBAAmB;YACtB,qBAAqB;YACrB,sBAAsB,EAClB,sBAAsB;gBACtB,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,oBAAoB,EAA+C,CAAC;SACvF,EACD,MAAM,CACT,CAAC;QA/BgB,WAAM,GAAN,MAAM,CAAkC;QAiC1D,uEAAuE;QACvE,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,wBAAwB,EAAE,CAAC;YACnD,MAAM,IAAI,KAAK,CAAC,iFAAiF,CAAC,CAAC;QACvG,CAAC;QAED,IAAI,CAAC,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,uBAAuB,EAAE,0BAA0B,CAAC,CAAC,CAAC;QAC5F,IAAI,mBAAmB,CAAC,MAAM;YAAE,IAAI,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC;QAEpF,IAAI,uBAAuB,IAAI,qBAAqB,EAAE,CAAC;YACnD,IAAI,CAAC,GAAG,CAAC,OAAO,CACZ,sGAAsG,CACzG,CAAC;QACN,CAAC;QAED,IAAI,CAAC,uBAAuB,GAAG,qBAAqB,GAAG,IAAI,CAAC;QAC5D,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,uBAAuB,GAAG,uBAAuB,CAAC;QACvD,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,CAAC;QACnD,IAAI,CAAC,8BAA8B,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,8BAA8B,CAAC,CAAC,CAAC;QACnF,IAAI,CAAC,0BAA0B,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,0BAA0B,CAAC,CAAC,CAAC;QAC3E,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,mBAAmB,GAAG;YACvB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,oBAAoB,CAAC,OAAO,EAAE,QAAS,CAAC;YACxE,GAAG,mBAAmB;SACzB,CAAC;QAEF,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,CAAC,wBAAwB,GAAG,wBAAwB,IAAI,IAAI,CAAC;QACrE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,wBAAwB,GAAG,KAAK,CAAC;QAC1C,CAAC;IACL,CAAC;IAES,oBAAoB;QAC1B,OAAO,eAAe,CAAC,MAAM,EAAmB;aAC3C,OAAO,CAAC;YACL,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC;SAC1C,CAAC;aACD,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;aACxD,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,6BAA6B,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC5E,CAAC;IAEO,KAAK,CAAC,eAAe,CACzB,eAAgC;QAEhC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAE7C,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YACzB,OAAO;gBACH,OAAO,EAAE,IAAI,KAAK,CAAC,OAAO,EAAE;oBACxB,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ;wBAC9B,IAAI,YAAY,KAAK,WAAW,EAAE,CAAC;4BAC/B,MAAM,IAAI,KAAK,CACX,+EAA+E,CAClF,CAAC;wBACN,CAAC;wBACD,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ,CAAC,CAAC;oBACvD,CAAC;iBACJ,CAAkC;gBACnC,IAAI,QAAQ;oBACR,MAAM,IAAI,KAAK,CAAC,sEAAsE,CAAC,CAAC;gBAC5F,CAAC;aACJ,CAAC;QACN,CAAC;QAED,MAAM,yBAAyB,GAAG,IAAI,CAAC,2BAA2B,CAAC,OAAO,CAAC,CAAC;QAE5E,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,UAAU,CAAC;QACxC,qEAAqE;QACrE,8CAA8C;QAC9C,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,kBAAkB,EAAE,eAAe,CAAC,CAAC;QACnE,SAAS,EAAE,CAAC;QAEZ,MAAM,0BAA0B,GAAG,IAAI,CAAC,2BAA2B,CAAC,OAAO,CAAC,CAAC;QAE7E,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,CAAC,eAAe,EAAE,yBAAyB,EAAE,0BAA0B,CAAC,CAAC;QAEhH,MAAM,QAAQ,GAAG,eAAe,CAAC,SAAS,EAAE,GAAG,CAAC;QAEhD,MAAM,YAAY,GAAG,MAAM,mBAAmB,CAC1C,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC,EAC/E,IAAI,CAAC,uBAAuB,EAC5B,2BAA2B,IAAI,CAAC,uBAAuB,GAAG,IAAI,WAAW,CAC5E,CAAC;QACF,SAAS,EAAE,CAAC;QAEZ,OAAO,CAAC,SAAS,GAAG,YAAY,EAAE,GAAG,CAAC;QACtC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,SAAS,CAAC;QAEvC,OAAO,EAAE,OAAO,EAAE,OAAwC,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC;IACzF,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAC7B,eAA2C;QAI3C,IAAI,eAAe,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YACzC,OAAO;gBACH,IAAI,WAAW;oBACX,MAAM,IAAI,KAAK,CAAC,yEAAyE,CAAC,CAAC;gBAC/F,CAAC;gBACD,IAAI,IAAI;oBACJ,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;gBACxF,CAAC;gBACD,IAAI,IAAI;oBACJ,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;gBACxF,CAAC;gBACD,IAAI,eAAe;oBACf,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAC;gBACjG,CAAC;gBACD,IAAI,gBAAgB;oBAChB,MAAM,IAAI,KAAK,CAAC,4EAA4E,CAAC,CAAC;gBAClG,CAAC;aACJ,CAAC;QACN,CAAC;QAED,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,mBAAmB,EAAE,eAAe,CAAC,CAAC;QACpE,SAAS,EAAE,CAAC;QAEZ,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,OAAO,EAAE,eAAe,CAAC,QAAQ,CAAC,CAAC;QAC5F,SAAS,EAAE,CAAC;QACZ,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAS,CAAC;QAClC,MAAM,WAAW,GAAG,MAAM,CAAC,WAAY,CAAC;QAExC,MAAM,eAAe,GAAG,KAAK,EAAE,QAAgB,EAAE,UAAmB,EAAE,EAAE;YACpE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;YAEhD,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACjC,MAAM,IAAI,KAAK,CAAC,aAAa,QAAQ,cAAc,CAAC,CAAC;YACzD,CAAC;QACL,CAAC,CAAC;QACF,MAAM,gBAAgB,GAAG,KAAK,EAAE,QAAiB,EAAE,SAAkB,EAAE,EAAE;YACrE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;YAEhD,IAAI,QAAQ,EAAE,CAAC;gBACX,MAAO,eAA+C,CAAC,eAAe,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;YAChG,CAAC;YAED,OAAO,CAAC,CAAC;QACb,CAAC,CAAC;QAEF,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,CAAC,sBAAsB,CAAC,eAAe,CAAC,OAAQ,EAAE,QAAQ,CAAC,MAAO,CAAC,CAAC;QAC5E,CAAC;QAED,IAAI,IAAI,CAAC,wBAAwB,EAAE,CAAC;YAChC,eAAe,CAAC,OAAQ,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC;QAC9D,CAAC;QAED,OAAO;YACH,IAAI,IAAI;gBACJ,IAAI,WAAW,CAAC,IAAI,KAAK,0BAA0B;oBAAE,OAAO,IAAI,CAAC;gBACjE,MAAM,UAAU,GAAG,MAAM,CAAC,IAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;gBAC/D,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAClC,CAAC;YACD,eAAe;YACf,gBAAgB;YAChB,WAAW;YACX,IAAI,EAAE,MAAM,CAAC,IAAI;SACpB,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,6BAA6B,CAAC,eAA4C;QACpF,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC;YAC3D,IAAI,KAAK;gBAAE,MAAM,IAAI,YAAY,CAAC,KAAK,CAAC,CAAC;QAC7C,CAAC;QACD,OAAO,EAAE,CAAC;IACd,CAAC;IAES,KAAK,CAAC,gBAAgB,CAAC,eAA4C;QACzE,IAAI,uBAAuB,CAAC,QAAQ,CAAC,eAAe,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;YACrE,MAAM,CAAC,GAAG,MAAM,eAAe,CAAC,gBAAgB,EAAE,CAAC;YAEnD,MAAM,cAAc,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAExF,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,OAAO,oBAAoB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3D,CAAC;QACL,CAAC;QAED,MAAM,kBAAkB;QACpB,wCAAwC;QACxC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,oBAAoB,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC;YACtD,CAAC,CAAC,wCAAwC;gBACxC,IAAI,CAAC,WAAY,CAAC,oBAAoB,CAAC;YACzC,CAAC,CAAC,oBAAoB,CAAC;QAE/B,IAAI,kBAAkB,CAAC,QAAQ,CAAC,eAAe,CAAC,QAAQ,CAAC,MAAO,CAAC,EAAE,CAAC;YAChE,OAAO,0BAA0B,eAAe,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;QACvE,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;OAGG;IACO,aAAa,CACnB,EAAE,OAAO,EAAE,OAAO,EAAmB,EACrC,cAAsB,EACtB,eAAuB;QAEvB,MAAM,aAAa,GAAG,OAAO,EAAE,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QAElE,MAAM,aAAa,GAAG,CAAC,aAAa,EAAE,cAAc,EAAE,eAAe,CAAC,CAAC;QAEvE,OAAO,YAAY,CAAC,OAAO,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;IACpD,CAAC;IAED;;;;OAIG;IACO,KAAK,CAAC,gBAAgB,CAAC,EAC7B,OAAO,EACP,OAAO,EACP,QAAQ,EACR,YAAY,GACS;QACrB,MAAM,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;QAEjE,IAAI,CAAC;YACD,OAAO,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC;QACrE,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,YAAY,KAAK,IAAI,CAAC,CAAC,WAAW,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;gBAC9D,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;gBACpC,OAAO,IAAI,QAAQ,EAAE,CAAC,CAAC,iEAAiE;YAC5F,CAAC;YAED,IAAI,IAAI,CAAC,YAAY,CAAC,CAAU,CAAC,EAAE,CAAC;gBAChC,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAU,CAAW,CAAC,CAAC;YAC5E,CAAC;iBAAM,CAAC;gBACJ,MAAM,CAAC,CAAC;YACZ,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,cAAc,CAAC,OAAuB,EAAE,QAAkB;QACtE,MAAM,EAAE,MAAM,EAAE,GAAG,QAAQ,CAAC;QAC5B,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,4BAA4B,CAAC,QAAQ,CAAC,CAAC;QACjE,MAAM,EAAE,QAAQ,EAAE,iBAAiB,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;QACnG,MAAM,WAAW,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QAEvC,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;YACjC,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;QAC1C,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,CAAC,0BAA0B,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACjE,MAAM,YAAY,GAAG,IAAI,CAAC,8BAA8B,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAErE,IAAI,CAAC,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,YAAY,EAAE,CAAC;YACnD,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,IAAI,EAAE,CAAC,CAAC,oGAAoG;YAEjJ,2DAA2D;YAC3D,gDAAgD;YAChD,IAAI,IAAI,KAAK,0BAA0B,EAAE,CAAC;gBACtC,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACvC,IAAI,EAAE,OAAO,EAAE,GAAG,aAAa,CAAC;gBAChC,IAAI,CAAC,OAAO;oBAAE,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,cAAc,EAAE,EAAE,EAAE,CAAC,CAAC;gBACtF,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,MAAM,OAAO,EAAE,CAAC,CAAC;YAC9C,CAAC;YAED,IAAI,YAAY,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,uCAAuC,CAAC,CAAC;YACtE,CAAC;YAED,8EAA8E;YAC9E,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,6BAA6B,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAChF,CAAC;aAAM,IAAI,uBAAuB,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAChD,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC,IAAI,EAAE,EAAE,CAAC;QAC3E,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,iBAAiB,CAAC,KAAK,EAAE,CAAC,CAAC;YAC1D,OAAO;gBACH,IAAI;gBACJ,QAAQ;gBACR,WAAW;aACd,CAAC;QACN,CAAC;IACL,CAAC;IAED;;OAEG;IACO,kBAAkB,CAAC,OAAuB,EAAE,OAAiB,EAAE,QAAiB;QACtF,MAAM,cAAc,GAAG;YACnB,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,QAAQ;YACR,OAAO,EAAE,IAAI,CAAC,uBAAuB;YACrC,SAAS,EAAE,IAAI,CAAC,wBAAwB,CAAC,CAAC,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,SAAS;YACzE,YAAY,EAAE,OAAO;YACrB,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,KAAK,EAAE;gBACH,kBAAkB,EAAE,CAAC,IAAI,CAAC,eAAe;aAC5C;YACD,IAAI,EAAE,SAA+B;SACxC,CAAC;QAEF,yHAAyH;QACzH,OAAO,CAAC,cAAc,CAAC,cAAc,CAAC,OAAQ,EAAE,QAAQ,CAAC,CAAC;QAE1D,4CAA4C;QAC5C,IAAI,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,CAAC;YACtC,cAAc,CAAC,KAAK,GAAG;gBACnB,GAAG,cAAc,CAAC,KAAK;gBACvB,kBAAkB,EAAE,KAAK;aAC5B,CAAC;QACN,CAAC;QAED,IAAI,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,cAAc,CAAC,IAAI,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;QAEvF,OAAO,cAAc,CAAC;IAC1B,CAAC;IAES,eAAe,CACrB,OAAuB,EACvB,QAAkB,EAClB,QAAwB;QAKxB,IAAI,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAC7B,QAAQ,GAAG,IAAI,CAAC,qBAAuC,CAAC;QAC5D,CAAC;aAAM,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,uBAAuB,EAAE,CAAC;YACnD,QAAQ,GAAG,IAAI,CAAC,uBAAyC,CAAC;QAC9D,CAAC;QAED,sDAAsD;QACtD,MAAM,IAAI,GAAG,MAAM,CAAC;QACpB,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAEnD,2DAA2D;QAC3D,+CAA+C;QAC/C,IAAI,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;QAE/D,+DAA+D;QAC/D,IAAI,KAAK,CAAC,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC;YACjC,MAAM,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YACxG,MAAM,aAAa,GAAG,QAAQ,CAAC,IAAI;gBAC/B,CAAC,CAAC,QAAQ,CAAC,KAAK,CACV,QAAQ,CAAC,IAAI,CACT,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAW,CAAC;qBACjC,IAAI,CAAC,YAAY,CAAC;qBAClB,IAAI,CAAC,YAAY,CAAC,CAC1B,CACJ;gBACH,CAAC,CAAC,IAAI,CAAC;YAEX,OAAO;gBACH,QAAQ,EAAE,IAAI,eAAe,CAAC,aAAoB,EAAE,QAAQ,CAAC;gBAC7D,QAAQ,EAAE,IAAI;aACjB,CAAC;QACN,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,YAAY,OAAO,CAAC,GAAG,8CAA8C,QAAQ,EAAE,CAAC,CAAC;IACrG,CAAC;IAED;;OAEG;IACO,yBAAyB,CAAC,mBAA4D;QAC5F,KAAK,MAAM,QAAQ,IAAI,mBAAmB,EAAE,CAAC;YACzC,IAAI,QAAQ,KAAK,KAAK,EAAE,CAAC;gBACrB,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBACtC,SAAS;YACb,CAAC;YAED,IAAI,CAAC;gBACD,MAAM,UAAU,GAAG,iBAAiB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBACrD,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;YACjD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,IAAI,KAAK,CAAC,2BAA2B,QAAQ,sCAAsC,CAAC,CAAC;YAC/F,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACO,qBAAqB,CAAC,OAAiB;QAC7C,OAAO,EAAE,OAAO,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,uBAAuB,GAAG,IAAI,WAAW,CAAC,CAAC;IAC/F,CAAC;IAEO,oBAAoB,CAAC,OAAuB,EAAE,QAAkB;QACpE,MAAM,EAAE,MAAM,EAAE,GAAG,QAAQ,CAAC;QAC5B,MAAM,EAAE,IAAI,EAAE,GAAG,4BAA4B,CAAC,QAAQ,CAAC,CAAC;QAExD,sEAAsE;QACtE,MAAM,kBAAkB,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1F,wDAAwD;QACxD,MAAM,sBAAsB,GAAG,MAAM,IAAI,GAAG,IAAI,kBAAkB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAEpF,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;YACvG,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;YACvB,MAAM,IAAI,KAAK,CACX,YAAY,OAAO,CAAC,GAAG,wBAAwB,IAAI,IAAI;gBACnD,YAAY,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,kCAAkC,CACnG,CAAC;QACN,CAAC;IACL,CAAC;IAED;;OAEG;IACK,iBAAiB,GAAG,KAAK,EAAE,OAAwB,EAAE,OAAiB,EAAE,YAAqB,EAAE,EAAE;QACrG,MAAM,IAAI,GAAG,yBAAyB,CAAC;YACnC,GAAI,OAAe;YACnB,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,YAAY,EAAE,MAAM;SACvB,CAAC,CAAC;QAEH,IAAI,YAAY,EAAE,CAAC;YACf,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC/B,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC/B,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QAC9C,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,CAC9C,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE;YAClB,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAS,CAAC,CAAC,CAAC,SAAS;YAChE,OAAO,EAAE,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;YAClC,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,kEAAkE;YAClE,MAAM,EAAE,MAAM;SACF,CAAC,EACjB;YACI,OAAO;YACP,OAAO,EAAE,IAAI,CAAC,OAAO;SACxB,CACJ,CAAC;QAEF,OAAO,QAAQ,CAAC;IACpB,CAAC,CAAC;;AAUN;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,UAAU,gBAAgB,CAG9B,MAAwC;IACtC,OAAO,MAAM,CAAC,MAAM,CAAU,MAAM,CAAC,CAAC;AAC1C,CAAC"}
|
package/internals/utils.d.ts
CHANGED
|
@@ -1,3 +1,8 @@
|
|
|
1
|
+
import type { HttpRequest, HttpRequestOptions } from '@crawlee/types';
|
|
2
|
+
/**
|
|
3
|
+
* Converts {@link HttpRequestOptions} to a {@link HttpRequest}.
|
|
4
|
+
*/
|
|
5
|
+
export declare function processHttpRequestOptions({ searchParams, form, json, username, password, ...request }: HttpRequestOptions): HttpRequest;
|
|
1
6
|
/**
|
|
2
7
|
* Gets parsed content type from response object
|
|
3
8
|
* @param response HTTP response object
|
package/internals/utils.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/internals/utils.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/internals/utils.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,WAAW,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAMtE;;GAEG;AACH,wBAAgB,yBAAyB,CAAC,EACtC,YAAY,EACZ,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,GAAG,OAAO,EACb,EAAE,kBAAkB,GAAG,WAAW,CAwClC;AAED;;;GAGG;AACH,wBAAgB,4BAA4B,CAAC,QAAQ,EAAE,QAAQ,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,cAAc,CAAA;CAAE,CAgC1G"}
|
package/internals/utils.js
CHANGED
|
@@ -1,7 +1,43 @@
|
|
|
1
1
|
import { extname } from 'node:path';
|
|
2
|
+
import { Readable } from 'node:stream';
|
|
3
|
+
import { applySearchParams } from '@crawlee/utils';
|
|
2
4
|
import contentTypeParser from 'content-type';
|
|
3
5
|
import mime from 'mime-types';
|
|
4
6
|
import ow, { ObjectPredicate } from 'ow';
|
|
7
|
+
/**
|
|
8
|
+
* Converts {@link HttpRequestOptions} to a {@link HttpRequest}.
|
|
9
|
+
*/
|
|
10
|
+
export function processHttpRequestOptions({ searchParams, form, json, username, password, ...request }) {
|
|
11
|
+
const url = new URL(request.url);
|
|
12
|
+
const headers = new Headers(request.headers);
|
|
13
|
+
applySearchParams(url, searchParams);
|
|
14
|
+
if ([request.body, form, json].filter((value) => value !== undefined).length > 1) {
|
|
15
|
+
throw new Error('At most one of `body`, `form` and `json` may be specified in sendRequest arguments');
|
|
16
|
+
}
|
|
17
|
+
const body = (() => {
|
|
18
|
+
if (form !== undefined) {
|
|
19
|
+
return Readable.from(new URLSearchParams(form).toString());
|
|
20
|
+
}
|
|
21
|
+
if (json !== undefined) {
|
|
22
|
+
return Readable.from(JSON.stringify(json));
|
|
23
|
+
}
|
|
24
|
+
if (request.body !== undefined) {
|
|
25
|
+
return Readable.from(request.body);
|
|
26
|
+
}
|
|
27
|
+
return undefined;
|
|
28
|
+
})();
|
|
29
|
+
if (form !== undefined && !headers.has('content-type')) {
|
|
30
|
+
headers.set('content-type', 'application/x-www-form-urlencoded');
|
|
31
|
+
}
|
|
32
|
+
if (json !== undefined && !headers.has('content-type')) {
|
|
33
|
+
headers.set('content-type', 'application/json');
|
|
34
|
+
}
|
|
35
|
+
if (username !== undefined || password !== undefined) {
|
|
36
|
+
const encodedAuth = Buffer.from(`${username ?? ''}:${password ?? ''}`).toString('base64');
|
|
37
|
+
headers.set('authorization', `Basic ${encodedAuth}`);
|
|
38
|
+
}
|
|
39
|
+
return { ...request, body, url, headers };
|
|
40
|
+
}
|
|
5
41
|
/**
|
|
6
42
|
* Gets parsed content type from response object
|
|
7
43
|
* @param response HTTP response object
|
package/internals/utils.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/internals/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/internals/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAGvC,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,iBAAiB,MAAM,cAAc,CAAC;AAC7C,OAAO,IAAI,MAAM,YAAY,CAAC;AAC9B,OAAO,EAAE,EAAE,EAAE,eAAe,EAAE,MAAM,IAAI,CAAC;AAEzC;;GAEG;AACH,MAAM,UAAU,yBAAyB,CAAC,EACtC,YAAY,EACZ,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,GAAG,OAAO,EACO;IACjB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACjC,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAE7C,iBAAiB,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IAErC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/E,MAAM,IAAI,KAAK,CAAC,oFAAoF,CAAC,CAAC;IAC1G,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,GAAG,EAAE;QACf,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACrB,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC/D,CAAC;QAED,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACrB,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/C,CAAC;QAED,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC7B,OAAO,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACvC,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC,CAAC,EAAE,CAAC;IAEL,IAAI,IAAI,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,mCAAmC,CAAC,CAAC;IACrE,CAAC;IAED,IAAI,IAAI,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,kBAAkB,CAAC,CAAC;IACpD,CAAC;IAED,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QACnD,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,IAAI,EAAE,IAAI,QAAQ,IAAI,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC1F,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,SAAS,WAAW,EAAE,CAAC,CAAC;IACzD,CAAC;IAED,OAAO,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC;AAC9C,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,4BAA4B,CAAC,QAAkB;IAC3D,EAAE,CACE,QAAQ,EACR,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC;QACnB,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG;QAClB,OAAO,EAAE,IAAI,eAAe,EAA2B;KAC1D,CAAC,CACL,CAAC;IAEF,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAC;IAClC,IAAI,iBAAiB,CAAC;IAEtB,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;QAC9B,IAAI,CAAC;YACD,iBAAiB,GAAG,iBAAiB,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAW,CAAC,CAAC;QACvF,CAAC;QAAC,MAAM,CAAC;YACL,4FAA4F;QAChG,CAAC;IACL,CAAC;IAED,qDAAqD;IACrD,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACrB,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/B,MAAM,sBAAsB,GACxB,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,IAAI,yCAAyC,CAAC,CAAC,0FAA0F;QAC1L,iBAAiB,GAAG,iBAAiB,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;IACxE,CAAC;IAED,OAAO;QACH,IAAI,EAAE,iBAAiB,CAAC,IAAI;QAC5B,OAAO,EAAE,iBAAiB,CAAC,UAAU,CAAC,OAAyB;KAClE,CAAC;AACN,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/http",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.23",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -49,13 +49,13 @@
|
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"@apify/timeout": "^0.3.2",
|
|
51
51
|
"@apify/utilities": "^2.15.5",
|
|
52
|
-
"@crawlee/basic": "4.0.0-beta.
|
|
53
|
-
"@crawlee/
|
|
54
|
-
"@crawlee/
|
|
52
|
+
"@crawlee/basic": "4.0.0-beta.23",
|
|
53
|
+
"@crawlee/http-client": "4.0.0-beta.23",
|
|
54
|
+
"@crawlee/types": "4.0.0-beta.23",
|
|
55
|
+
"@crawlee/utils": "4.0.0-beta.23",
|
|
55
56
|
"@types/content-type": "^1.1.8",
|
|
56
57
|
"cheerio": "^1.0.0",
|
|
57
58
|
"content-type": "^1.0.5",
|
|
58
|
-
"got-scraping": "^4.1.1",
|
|
59
59
|
"iconv-lite": "^0.7.0",
|
|
60
60
|
"mime-types": "^3.0.1",
|
|
61
61
|
"ow": "^2.0.0",
|
|
@@ -69,5 +69,5 @@
|
|
|
69
69
|
}
|
|
70
70
|
}
|
|
71
71
|
},
|
|
72
|
-
"gitHead": "
|
|
72
|
+
"gitHead": "929fd9374b74ee5e9720b3864b39326f014cd45b"
|
|
73
73
|
}
|