@crawlee/http 4.0.0-beta.17 → 4.0.0-beta.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (9) hide show
  1. package/internals/file-download.d.ts +2 -2
  2. package/internals/file-download.d.ts.map +1 -1
  3. package/internals/file-download.js +4 -6
  4. package/internals/file-download.js.map +1 -1
  5. package/internals/http-crawler.d.ts +31 -97
  6. package/internals/http-crawler.d.ts.map +1 -1
  7. package/internals/http-crawler.js +41 -59
  8. package/internals/http-crawler.js.map +1 -1
  9. package/package.json +5 -5
package/internals/file-download.d.ts CHANGED
@@ -55,11 +55,11 @@ export declare function ByteCounterStream({ logTransferredBytes, loggingInterval
55
55
  *
56
56
  * The crawler finishes when there are no more {@link Request} objects to crawl.
57
57
  *
58
- * We can use the `preNavigationHooks` to adjust `gotOptions`:
58
+ * We can use the `preNavigationHooks` to adjust the crawling context before the request is made:
59
59
  *
60
60
  * ```
61
61
  * preNavigationHooks: [
62
- * (crawlingContext, gotOptions) => {
62
+ * (crawlingContext) => {
63
63
  * // ...
64
64
  * },
65
65
  * ]
package/internals/file-download.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"file-download.d.ts","sourceRoot":"","sources":["../../src/internals/file-download.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAGxC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAmB,MAAM,gBAAgB,CAAC;AAC/D,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAC7E,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEjD,OAAO,KAAK,EAAE,YAAY,EAAE,sBAAsB,EAAE,gBAAgB,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAIxH,MAAM,MAAM,wBAAwB,CAChC,QAAQ,SAAS,UAAU,GAAG,GAAG,IACjC,YAAY,CAAC,2BAA2B,CAAC,QAAQ,CAAC,CAAC,CAAC;AAExD,MAAM,MAAM,gBAAgB,CACxB,QAAQ,SAAS,UAAU,GAAG,GAAG,IACjC,gBAAgB,CAAC,2BAA2B,CAAC,QAAQ,CAAC,CAAC,CAAC;AAE5D,MAAM,WAAW,2BAA2B,CACxC,QAAQ,SAAS,UAAU,GAAG,GAAG,CACnC,SAAQ,eAAe,CAAC,QAAQ,CAAC;IAC/B,OAAO,EAAE,aAAa,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC1C,QAAQ,EAAE,QAAQ,CAAC;IACnB,WAAW,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,cAAc,CAAA;KAAE,CAAC;CAC3D;AAED,MAAM,MAAM,0BAA0B,CAClC,QAAQ,SAAS,UAAU,GAAG,GAAG,IACjC,cAAc,CAAC,2BAA2B,CAAC,QAAQ,CAAC,CAAC,CAAC;AAE1D;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAAC,EAC/B,YAAY,EACZ,eAAsB,EACtB,qBAAqB,EAAE,uBAA6B,GACvD,EAAE;IACC,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAClC,GAAG,SAAS,CA4BZ;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,EAC9B,mBAAmB,EACnB,eAAsB,GACzB,EAAE;IACC,mBAAmB,EAAE,CAAC,gBAAgB,EAAE,MAAM,KAAK,IAAI,CAAC;IACxD,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B,GAAG,SAAS,CAoBZ;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AACH,qBAAa,YAAa,SAAQ,YAAY,CAAC,2BAA2B,CAAC;gBAE3D,OAAO,GAAE,mBAAmB,CAAC,2BAA2B,CAAM;YAa5D,gBAAgB;CAmBjC;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,gBAAgB,CAC5B,OAAO,SAAS,2BAA2B,GAAG,2BAA2B,EACzE,QAAQ,SAAS,UAAU,GAAG,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAC1E,MAAM,CAAC,EAAE,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,mDAEzC"}
1
+ {"version":3,"file":"file-download.d.ts","sourceRoot":"","sources":["../../src/internals/file-download.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAGxC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAmB,MAAM,gBAAgB,CAAC;AAC/D,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAC7E,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEjD,OAAO,KAAK,EAAE,YAAY,EAAE,sBAAsB,EAAE,gBAAgB,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAIxH,MAAM,MAAM,wBAAwB,CAChC,QAAQ,SAAS,UAAU,GAAG,GAAG,IACjC,YAAY,CAAC,2BAA2B,CAAC,QAAQ,CAAC,CAAC,CAAC;AAExD,MAAM,MAAM,gBAAgB,CACxB,QAAQ,SAAS,UAAU,GAAG,GAAG,IACjC,gBAAgB,CAAC,2BAA2B,CAAC,QAAQ,CAAC,CAAC,CAAC;AAE5D,MAAM,WAAW,2BAA2B,CACxC,QAAQ,SAAS,UAAU,GAAG,GAAG,CACnC,SAAQ,eAAe,CAAC,QAAQ,CAAC;IAC/B,OAAO,EAAE,aAAa,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC1C,QAAQ,EAAE,QAAQ,CAAC;IACnB,WAAW,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,cAAc,CAAA;KAAE,CAAC;CAC3D;AAED,MAAM,MAAM,0BAA0B,CAClC,QAAQ,SAAS,UAAU,GAAG,GAAG,IACjC,cAAc,CAAC,2BAA2B,CAAC,QAAQ,CAAC,CAAC,CAAC;AAE1D;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAAC,EAC/B,YAAY,EACZ,eAAsB,EACtB,qBAAqB,EAAE,uBAA6B,GACvD,EAAE;IACC,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAClC,GAAG,SAAS,CA4BZ;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,EAC9B,mBAAmB,EACnB,eAAsB,GACzB,EAAE;IACC,mBAAmB,EAAE,CAAC,gBAAgB,EAAE,MAAM,KAAK,IAAI,CAAC;IACxD,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B,GAAG,SAAS,CAoBZ;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AACH,qBAAa,YAAa,SAAQ,YAAY,CAAC,2BAA2B,CAAC;gBAE3D,OAAO,GAAE,mBAAmB,CAAC,2BAA2B,CAAM;YAa5D,gBAAgB;CAiBjC;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,gBAAgB,CAC5B,OAAO,SAAS,2BAA2B,GAAG,2BAA2B,EACzE,QAAQ,SAAS,UAAU,GAAG,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAC1E,MAAM,CAAC,EAAE,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,mDAEzC"}
package/internals/file-download.js CHANGED
@@ -76,11 +76,11 @@ export function ByteCounterStream({ logTransferredBytes, loggingInterval = 5000,
76
76
  *
77
77
  * The crawler finishes when there are no more {@link Request} objects to crawl.
78
78
  *
79
- * We can use the `preNavigationHooks` to adjust `gotOptions`:
79
+ * We can use the `preNavigationHooks` to adjust the crawling context before the request is made:
80
80
  *
81
81
  * ```
82
82
  * preNavigationHooks: [
83
- * (crawlingContext, gotOptions) => {
83
+ * (crawlingContext) => {
84
84
  * // ...
85
85
  * },
86
86
  * ]
@@ -118,10 +118,8 @@ export class FileDownload extends BasicCrawler {
118
118
  });
119
119
  }
120
120
  async initiateDownload(context) {
121
- const response = await this.httpClient.stream({
122
- url: context.request.url,
123
- timeout: { request: undefined },
124
- proxyUrl: context.proxyInfo?.url,
121
+ const response = await this.httpClient.stream(context.request.intoFetchAPIRequest(), {
122
+ session: context.session,
125
123
  });
126
124
  const { type, charset: encoding } = parseContentTypeFromResponse(response);
127
125
  context.request.url = response.url;
package/internals/file-download.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"file-download.js","sourceRoot":"","sources":["../../src/internals/file-download.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAGhD,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAK/D,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAE,4BAA4B,EAAE,MAAM,YAAY,CAAC;AAsB1D;;;;;;;;GAQG;AACH,MAAM,UAAU,kBAAkB,CAAC,EAC/B,YAAY,EACZ,eAAe,GAAG,IAAI,EACtB,qBAAqB,EAAE,uBAAuB,GAAG,GAAG,GAKvD;IACG,IAAI,SAAS,GAA2C,EAAE,CAAC;IAE3D,MAAM,aAAa,GAAG,WAAW,CAAC,GAAG,EAAE;QACnC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEvB,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,GAAG,GAAG,QAAQ,CAAC,SAAS,GAAG,eAAe,CAAC,CAAC;QACvF,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,QAAQ,EAAE,EAAE,CAAC,GAAG,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAChF,MAAM,OAAO,GAAG,CAAC,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;QAE9D,IAAI,UAAU,GAAG,IAAI,GAAG,OAAO,GAAG,YAAY,EAAE,CAAC;YAC7C,aAAa,CAAC,aAAa,CAAC,CAAC;YAC7B,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC,CAAC;QAC1E,CAAC;IACL,CAAC,EAAE,uBAAuB,CAAC,CAAC;IAE5B,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;QACzB,SAAS,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE;YAC9B,SAAS,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;YAC/D,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC1B,CAAC;QACD,KAAK,EAAE,CAAC,QAAQ,EAAE,EAAE;YAChB,aAAa,CAAC,aAAa,CAAC,CAAC;YAC7B,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAClB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB,CAAC,EAC9B,mBAAmB,EACnB,eAAe,GAAG,IAAI,GAIzB;IACG,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,IAAI,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,OAAO,IAAI,SAAS,CAAC;QACjB,SAAS,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE;YAC9B,gBAAgB,IAAI,KAAK,CAAC,MAAM,CAAC;YAEjC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,GAAG,eAAe,EAAE,CAAC;gBAC7C,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACzB,mBAAmB,CAAC,gBAAgB,CAAC,CAAC;YAC1C,CAAC;YAED,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC1B,CAAC;QACD,KAAK,EAAE,CAAC,QAAQ,EAAE,EAAE;YAChB,mBAAmB,CAAC,gBAAgB,CAAC,CAAC;YACtC,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;AACP,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AACH,MAAM,OAAO,YAAa,SAAQ,YAAyC;IACvE,aAAa;IACb,YAAY,UAA4D,EAAE;QACtE,KAAK,CAAC;YACF,GAAG,OAAO;YACV,sBAAsB,EAAE,GAAG,EAAE,CACzB,eAAe,CAAC,MAAM,EAAmB,CAAC,OAAO,CAAC;gBAC9C,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC;gBACzD,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;oBACvB,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAW,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC/F,CAAC;aACJ,CAAC;SACT,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,OAAwB;QACnD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;YAC1C,GAAG,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG;YACxB,OAAO,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE;YAC/B,QAAQ,EAAE,OAAO,CAAC,SAAS,EAAE,GAAG;SACnC,CAAC,CAAC;QAEH,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,4BAA4B,CAAC,QAAQ,CAAC,CAAC;QAE3E,OAAO,CAAC,OAAO,CAAC,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC;QAEnC,MAAM,gBAAgB,GAAG;YACrB,OAAO,EAAE,OAAO,CAAC,OAAiC;YAClD,QAAQ;YACR,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;SAClC,CAAC;QAEF,OAAO,gBAAgB,CAAC;IAC5B,CAAC;CACJ;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,UAAU,gBAAgB,CAG9B,MAAwC;IACtC,OAAO,MAAM,CAAC,MAAM,CAAU,MAAM,CAAC,CAAC;AAC1C,CAAC"}
1
+ {"version":3,"file":"file-download.js","sourceRoot":"","sources":["../../src/internals/file-download.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAGhD,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAK/D,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAE,4BAA4B,EAAE,MAAM,YAAY,CAAC;AAsB1D;;;;;;;;GAQG;AACH,MAAM,UAAU,kBAAkB,CAAC,EAC/B,YAAY,EACZ,eAAe,GAAG,IAAI,EACtB,qBAAqB,EAAE,uBAAuB,GAAG,GAAG,GAKvD;IACG,IAAI,SAAS,GAA2C,EAAE,CAAC;IAE3D,MAAM,aAAa,GAAG,WAAW,CAAC,GAAG,EAAE;QACnC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEvB,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,GAAG,GAAG,QAAQ,CAAC,SAAS,GAAG,eAAe,CAAC,CAAC;QACvF,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,QAAQ,EAAE,EAAE,CAAC,GAAG,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAChF,MAAM,OAAO,GAAG,CAAC,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;QAE9D,IAAI,UAAU,GAAG,IAAI,GAAG,OAAO,GAAG,YAAY,EAAE,CAAC;YAC7C,aAAa,CAAC,aAAa,CAAC,CAAC;YAC7B,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC,CAAC;QAC1E,CAAC;IACL,CAAC,EAAE,uBAAuB,CAAC,CAAC;IAE5B,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;QACzB,SAAS,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE;YAC9B,SAAS,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;YAC/D,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC1B,CAAC;QACD,KAAK,EAAE,CAAC,QAAQ,EAAE,EAAE;YAChB,aAAa,CAAC,aAAa,CAAC,CAAC;YAC7B,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAClB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB,CAAC,EAC9B,mBAAmB,EACnB,eAAe,GAAG,IAAI,GAIzB;IACG,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,IAAI,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,OAAO,IAAI,SAAS,CAAC;QACjB,SAAS,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE;YAC9B,gBAAgB,IAAI,KAAK,CAAC,MAAM,CAAC;YAEjC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,GAAG,eAAe,EAAE,CAAC;gBAC7C,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACzB,mBAAmB,CAAC,gBAAgB,CAAC,CAAC;YAC1C,CAAC;YAED,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC1B,CAAC;QACD,KAAK,EAAE,CAAC,QAAQ,EAAE,EAAE;YAChB,mBAAmB,CAAC,gBAAgB,CAAC,CAAC;YACtC,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;AACP,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AACH,MAAM,OAAO,YAAa,SAAQ,YAAyC;IACvE,aAAa;IACb,YAAY,UAA4D,EAAE;QACtE,KAAK,CAAC;YACF,GAAG,OAAO;YACV,sBAAsB,EAAE,GAAG,EAAE,CACzB,eAAe,CAAC,MAAM,EAAmB,CAAC,OAAO,CAAC;gBAC9C,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC;gBACzD,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;oBACvB,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAW,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC/F,CAAC;aACJ,CAAC;SACT,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,OAAwB;QACnD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,mBAAmB,EAAE,EAAE;YACjF,OAAO,EAAE,OAAO,CAAC,OAAO;SAC3B,CAAC,CAAC;QAEH,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,4BAA4B,CAAC,QAAQ,CAAC,CAAC;QAE3E,OAAO,CAAC,OAAO,CAAC,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC;QAEnC,MAAM,gBAAgB,GAAG;YACrB,OAAO,EAAE,OAAO,CAAC,OAAiC;YAClD,QAAQ;YACR,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;SAClC,CAAC;QAEF,OAAO,gBAAgB,CAAC;IAC5B,CAAC;CACJ;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,UAAU,gBAAgB,CAG9B,MAAwC;IACtC,OAAO,MAAM,CAAC,MAAM,CAAU,MAAM,CAAC,CAAC;AAC1C,CAAC"}
package/internals/http-crawler.d.ts CHANGED
@@ -1,21 +1,10 @@
1
- import type { IncomingMessage } from 'node:http';
2
- import { Readable } from 'node:stream';
3
- import type { BasicCrawlerOptions, CrawlingContext, ErrorHandler, GetUserDataFromRequest, Request, RequestHandler, RequireContextPipeline, RouterRoutes, Session } from '@crawlee/basic';
1
+ import type { BasicCrawlerOptions, CrawlingContext, ErrorHandler, GetUserDataFromRequest, Request as CrawleeRequest, RequestHandler, RequireContextPipeline, RouterRoutes, Session } from '@crawlee/basic';
4
2
  import { BasicCrawler, Configuration, ContextPipeline } from '@crawlee/basic';
5
- import type { HttpResponse, LoadedRequest } from '@crawlee/core';
3
+ import type { LoadedRequest } from '@crawlee/core';
6
4
  import type { Awaitable, Dictionary } from '@crawlee/types';
7
5
  import { type CheerioRoot } from '@crawlee/utils';
8
6
  import type { RequestLike, ResponseLike } from 'content-type';
9
- // @ts-ignore optional peer dependency or compatibility with es2022
10
- import type { Method, OptionsInit } from 'got-scraping';
11
7
  import type { JsonValue } from 'type-fest';
12
- /**
13
- * TODO exists for BC within HttpCrawler - replace completely with StreamingHttpResponse in 4.0
14
- * @internal
15
- */
16
- export type PlainResponse = Omit<HttpResponse, 'body'> & IncomingMessage & {
17
- body?: unknown;
18
- };
19
8
  export type HttpErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
20
9
  JSONData extends JsonValue = any> = ErrorHandler<HttpCrawlingContext<UserData, JSONData>>;
21
10
  export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension> extends BasicCrawlerOptions<Context, ContextExtension, ExtendedContext> {
@@ -29,12 +18,12 @@ export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext
29
18
  ignoreSslErrors?: boolean;
30
19
  /**
31
20
  * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
32
- * or browser properties before navigation. The function accepts two parameters, `crawlingContext` and `gotOptions`,
33
- * which are passed to the `requestAsBrowser()` function the crawler calls to navigate.
21
+ * or browser properties before navigation. The function accepts one parameter `crawlingContext`,
22
+ * which is passed to the `requestAsBrowser()` function the crawler calls to navigate.
34
23
  * Example:
35
24
  * ```
36
25
  * preNavigationHooks: [
37
- * async (crawlingContext, gotOptions) => {
26
+ * async (crawlingContext) => {
38
27
  * // ...
39
28
  * },
40
29
  * ]
@@ -105,14 +94,14 @@ export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext
105
94
  /**
106
95
  * @internal
107
96
  */
108
- export type InternalHttpHook<Context> = (crawlingContext: Context, gotOptions: OptionsInit) => Awaitable<void>;
97
+ export type InternalHttpHook<Context> = (crawlingContext: Context) => Awaitable<void>;
109
98
  export type HttpHook<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
110
99
  JSONData extends JsonValue = any> = InternalHttpHook<HttpCrawlingContext<UserData, JSONData>>;
111
100
  interface CrawlingContextWithReponse<UserData extends Dictionary = any> extends CrawlingContext<UserData> {
112
101
  /**
113
102
  * The request object that was successfully loaded and navigated to, including the {@link Request.loadedUrl|`loadedUrl`} property.
114
103
  */
115
- request: LoadedRequest<Request<UserData>>;
104
+ request: LoadedRequest<CrawleeRequest<UserData>>;
116
105
  /**
117
106
  * The HTTP response object containing status code, headers, and other response metadata.
118
107
  */
@@ -193,11 +182,11 @@ JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData,
193
182
  *
194
183
  * The crawler finishes when there are no more {@link Request} objects to crawl.
195
184
  *
196
- * We can use the `preNavigationHooks` to adjust `gotOptions`:
185
+ * We can use the `preNavigationHooks` to adjust the crawling context before the request is made:
197
186
  *
198
187
  * ```javascript
199
188
  * preNavigationHooks: [
200
- * (crawlingContext, gotOptions) => {
189
+ * (crawlingContext) => {
201
190
  * // ...
202
191
  * },
203
192
  * ]
@@ -343,19 +332,20 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
343
332
  private handleBlockedRequestByContent;
344
333
  protected isRequestBlocked(crawlingContext: InternalHttpCrawlingContext): Promise<string | false>;
345
334
  /**
346
- * Sets the cookie header to `gotOptions` based on the provided request and session headers, as well as any changes that occurred due to hooks.
335
+ * Returns the `Cookie` header value based on the current context and
336
+ * any changes that occurred in the navigation hooks.
347
337
  */
348
- protected _applyCookies({ session, request }: CrawlingContext, gotOptions: OptionsInit, preHookCookies: string, postHookCookies: string): void;
338
+ protected _applyCookies({ session, request }: CrawlingContext, preHookCookies: string, postHookCookies: string): string;
349
339
  /**
350
340
  * Function to make the HTTP request. It performs optimizations
351
341
  * on the request such as only downloading the request body if the
352
342
  * received content type matches text/html, application/xml, application/xhtml+xml.
353
343
  */
354
- protected _requestFunction({ request, session, proxyUrl, gotOptions, }: RequestFunctionOptions): Promise<Response>;
344
+ protected _requestFunction({ request, session, proxyUrl, cookieString, }: RequestFunctionOptions): Promise<Response>;
355
345
  /**
356
346
  * Encodes and parses response according to the provided content type
357
347
  */
358
- protected _parseResponse(request: Request, response: Response): Promise<{
348
+ protected _parseResponse(request: CrawleeRequest, response: Response): Promise<{
359
349
  response: Response;
360
350
  contentType: {
361
351
  type: string;
@@ -373,80 +363,24 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
373
363
  /**
374
364
  * Combines the provided `requestOptions` with mandatory (non-overridable) values.
375
365
  */
376
- protected _getRequestOptions(request: Request, session?: Session, proxyUrl?: string, gotOptions?: OptionsInit): {
377
- url?: string | URL | undefined;
378
- // @ts-ignore optional peer dependency or compatibility with es2022
379
- headers?: import("got-scraping").Headers | undefined;
380
- // @ts-ignore optional peer dependency or compatibility with es2022
381
- body?: string | Buffer | Readable | Generator | AsyncGenerator | import("form-data-encoder").FormDataLike | undefined;
382
- json?: unknown;
383
- // @ts-ignore optional peer dependency or compatibility with es2022
384
- request?: import("got-scraping").RequestFunction | undefined;
385
- // @ts-ignore optional peer dependency or compatibility with es2022
386
- agent?: import("got-scraping").Agents | undefined;
387
- // @ts-ignore optional peer dependency or compatibility with es2022
388
- h2session?: import("http2").ClientHttp2Session | undefined;
389
- decompress?: boolean | undefined;
390
- // @ts-ignore optional peer dependency or compatibility with es2022
391
- timeout?: import("got-scraping").Delays | undefined;
392
- prefixUrl?: string | URL | undefined;
393
- form?: Record<string, any> | undefined;
394
- // @ts-ignore optional peer dependency or compatibility with es2022
395
- cookieJar?: import("got-scraping").PromiseCookieJar | import("got-scraping").ToughCookieJar | undefined;
396
- signal?: AbortSignal | undefined;
397
- ignoreInvalidCookies?: boolean | undefined;
398
- // @ts-ignore optional peer dependency or compatibility with es2022
399
- searchParams?: string | import("got-scraping").SearchParameters | URLSearchParams | undefined;
400
- // @ts-ignore optional peer dependency or compatibility with es2022
401
- dnsLookup?: import("cacheable-lookup").default["lookup"] | undefined;
366
+ protected _getRequestOptions(request: CrawleeRequest, session?: Session, proxyUrl?: string): {
367
+ url: string;
402
368
  // @ts-ignore optional peer dependency or compatibility with es2022
403
- dnsCache?: import("cacheable-lookup").default | boolean | undefined;
404
- context?: Record<string, unknown> | undefined;
405
- // @ts-ignore optional peer dependency or compatibility with es2022
406
- followRedirect?: boolean | ((response: import("got-scraping").PlainResponse) => boolean) | undefined;
407
- maxRedirects?: number | undefined;
408
- // @ts-ignore optional peer dependency or compatibility with es2022
409
- cache?: string | import("cacheable-request").StorageAdapter | boolean | undefined;
410
- throwHttpErrors?: boolean | undefined;
411
- username?: string | undefined;
412
- password?: string | undefined;
413
- http2?: boolean | undefined;
414
- allowGetBody?: boolean | undefined;
415
- methodRewriting?: boolean | undefined;
416
- // @ts-ignore optional peer dependency or compatibility with es2022
417
- dnsLookupIpVersion?: import("got-scraping").DnsLookupIpVersion;
418
- // @ts-ignore optional peer dependency or compatibility with es2022
419
- parseJson?: import("got-scraping").ParseJsonFunction | undefined;
420
- // @ts-ignore optional peer dependency or compatibility with es2022
421
- stringifyJson?: import("got-scraping").StringifyJsonFunction | undefined;
422
- localAddress?: string | undefined;
423
- method?: Method | undefined;
424
- // @ts-ignore optional peer dependency or compatibility with es2022
425
- createConnection?: import("got-scraping").CreateConnectionFunction | undefined;
426
- // @ts-ignore optional peer dependency or compatibility with es2022
427
- cacheOptions?: import("got-scraping").CacheOptions | undefined;
428
- // @ts-ignore optional peer dependency or compatibility with es2022
429
- https?: import("got-scraping").HttpsOptions | undefined;
430
- encoding?: BufferEncoding | undefined;
431
- resolveBodyOnly?: boolean | undefined;
432
- isStream?: boolean | undefined;
433
- // @ts-ignore optional peer dependency or compatibility with es2022
434
- responseType?: import("got-scraping").ResponseType | undefined;
435
- // @ts-ignore optional peer dependency or compatibility with es2022
436
- pagination?: import("got-scraping").PaginationOptions<unknown, unknown> | undefined;
437
- setHost?: boolean | undefined;
438
- maxHeaderSize?: number | undefined;
439
- enableUnixSockets?: boolean | undefined;
440
- } & {
441
- // @ts-ignore optional peer dependency or compatibility with es2022
442
- hooks?: Partial<import("got-scraping").Hooks>;
443
- // @ts-ignore optional peer dependency or compatibility with es2022
444
- retry?: Partial<import("got-scraping").RetryOptions>;
369
+ method: import("@crawlee/types").AllowedHttpMethods;
370
+ proxyUrl: string | undefined;
371
+ timeout: {
372
+ request: number;
373
+ };
445
374
  // @ts-ignore optional peer dependency or compatibility with es2022
446
- } & import("got-scraping").Context & Required<Pick<OptionsInit, "url">> & {
447
- isStream: true;
375
+ cookieJar: import("tough-cookie").CookieJar | undefined;
376
+ sessionToken: Session | undefined;
377
+ headers: Record<string, string> | undefined;
378
+ https: {
379
+ rejectUnauthorized: boolean;
380
+ };
381
+ body: string | undefined;
448
382
  };
449
- protected _encodeResponse(request: Request, response: Response, encoding: BufferEncoding): {
383
+ protected _encodeResponse(request: CrawleeRequest, response: Response, encoding: BufferEncoding): {
450
384
  encoding: BufferEncoding;
451
385
  response: Response;
452
386
  };
@@ -465,10 +399,10 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
465
399
  private _requestAsBrowser;
466
400
  }
467
401
  interface RequestFunctionOptions {
468
- request: Request;
402
+ request: CrawleeRequest;
469
403
  session?: Session;
470
404
  proxyUrl?: string;
471
- gotOptions: OptionsInit;
405
+ cookieString?: string;
472
406
  }
473
407
  /**
474
408
  * Creates new {@link Router} instance that works based on request labels.
package/internals/http-crawler.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"http-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/http-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAGvC,OAAO,KAAK,EAER,mBAAmB,EACnB,eAAe,EACf,YAAY,EACZ,sBAAsB,EACtB,OAAO,EACP,cAAc,EACd,sBAAsB,EACtB,YAAY,EACZ,OAAO,EACV,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACH,YAAY,EAEZ,aAAa,EACb,eAAe,EAOlB,MAAM,gBAAgB,CAAC;AACxB,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACjE,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5D,OAAO,EAAE,KAAK,WAAW,EAAuB,MAAM,gBAAgB,CAAC;AAEvE,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE9D,OAAO,KAAK,EAAE,MAAM,EAAE,WAAW,EAAqC,MAAM,cAAc,CAAC;AAG3F,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAQ3C;;;GAGG;AACH,MAAM,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,GAClD,eAAe,GAAG;IACd,IAAI,CAAC,EAAE,OAAO,CAAC;CAClB,CAAC;AAkBN,MAAM,MAAM,gBAAgB,CACxB,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,IAChC,YAAY,CAAC,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE1D,MAAM,WAAW,kBAAkB,CAC/B,OAAO,SAAS,2BAA2B,GAAG,2BAA2B,EACzE,gBAAgB,GAAG,UAAU,CAAC,KAAK,CAAC,EACpC,eAAe,SAAS,OAAO,GAAG,OAAO,GAAG,gBAAgB,CAC9D,SAAQ,mBAAmB,CAAC,OAAO,EAAE,gBAAgB,EAAE,eAAe,CAAC;IACrE;;OAEG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;OAEG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;;;;;;;;;;;;;;OAeG;IACH,kBAAkB,CAAC,EAAE,gBAAgB,CAAC,eAAe,CAAC,EAAE,CAAC;IAEzD;;;;;;;;;;;OAWG;IACH,mBAAmB,CAAC,EAAE,CAAC,CAAC,eAAe,EAAE,0BAA0B,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;IAE3F;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE/B;;;;;;;;;;OAUG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAEjC;;;;;;;;OAQG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;;;OAKG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAC;IAEnC;;;OAGG;IACH,0BAA0B,CAAC,EAAE,MAAM,EAAE,CAAC;IAEtC;;;OAGG;IACH,8BAA8B,CAAC,EAAE,MAAM,EAAE,CAAC;CAC7C;AAED;;GAEG;AACH,MAAM,MAAM,gBAAgB,CAAC,OAAO,IAAI,CAAC,eAAe,EAAE,OAAO,EAAE,UAAU,EAAE,WAAW,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAE/G,MAAM,MAAM,QAAQ,CAChB,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,IAChC,gBAAgB,CAAC,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE9D,UAAU,0BAA0B,CAChC,QAAQ,SAAS,UAAU,GAAG,GAAG,CACnC,SAAQ,eAAe,CAAC,QAAQ,CAAC;IAC/B;;OAEG;IACH,OAAO,EAAE,aAAa,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IAE1C;;OAEG;IACH,QAAQ,EAAE,QAAQ,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,2BAA2B,CACxC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,CAClC,SAAQ,0BAA0B,CAAC,QAAQ,CAAC;IAC1C;;;;;OAKG;IACH,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;IAEtB;;OAEG;IACH,IAAI,EAAE,QAAQ,CAAC;IAEf;;OAEG;IACH,WAAW,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,cAAc,CAAA;KAAE,CAAC;IAExD;;;;;;;;;;;OAWG;IACH,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErE;;;;;;;;;;;OAWG;IACH,gBAAgB,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;CACjF;AAED,MAAM,WAAW,mBAAmB,CAAC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,QAAQ,SAAS,SAAS,GAAG,GAAG,CACpG,SAAQ,2BAA2B,CAAC,QAAQ,EAAE,QAAQ,CAAC;CAAG;AAE9D,MAAM,MAAM,kBAAkB,CAC1B,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,IAChC,cAAc,CAAC,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkEG;AACH,qBAAa,WAAW,CACpB,OAAO,SAAS,2BAA2B,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,2BAA2B,EACnF,gBAAgB,GAAG,UAAU,CAAC,KAAK,CAAC,EACpC,eAAe,SAAS,OAAO,GAAG,OAAO,GAAG,gBAAgB,CAC9D,SAAQ,YAAY,CAAC,OAAO,EAAE,gBAAgB,EAAE,eAAe,CAAC;aAmCxC,MAAM;IAlC5B,SAAS,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,eAAe,CAAC,EAAE,CAAC;IAClE,SAAS,CAAC,mBAAmB,EAAE,CAAC,CAAC,eAAe,EAAE,0BAA0B,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;IACpG,SAAS,CAAC,wBAAwB,EAAE,OAAO,CAAC;IAC5C,SAAS,CAAC,uBAAuB,EAAE,MAAM,CAAC;IAC1C,SAAS,CAAC,eAAe,EAAE,OAAO,CAAC;IACnC,SAAS,CAAC,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAC3C,SAAS,CAAC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IACzC,SAAS,CAAC,8BAA8B,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACtD,SAAS,CAAC,0BAA0B,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAClD,SAAS,CAAC,QAAQ,CAAC,kBAAkB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAEnD,iBAA0B,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAepC;IAEF;;OAEG;gBAEC,OAAO,GAAE,kBAAkB,CAAC,OAAO,EAAE,gBAAgB,EAAE,eAAe,CAAC,GACnE,sBAAsB,CAAC,2BAA2B,EAAE,OAAO,CAAa,EAC1D,MAAM,gBAAkC;IAkE9D,SAAS,CAAC,oBAAoB,IAAI,eAAe,CAAC,eAAe,EAAE,2BAA2B,CAAC;YASjF,eAAe;YAmDf,mBAAmB;YAuEnB,6BAA6B;cAQ3B,gBAAgB,CAAC,eAAe,EAAE,2BAA2B,GAAG,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC;IAyBvG;;OAEG;IACH,SAAS,CAAC,aAAa,CACnB,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,eAAe,EACrC,UAAU,EAAE,WAAW,EACvB,cAAc,EAAE,MAAM,EACtB,eAAe,EAAE,MAAM;IAkD3B;;;;OAIG;cACa,gBAAgB,CAAC,EAC7B,OAAO,EACP,OAAO,EACP,QAAQ,EACR,UAAU,GACb,EAAE,sBAAsB,GAAG,OAAO,CAAC,QAAQ,CAAC;IAwB7C;;OAEG;cACa,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ;;;;;;;;;;;;;;;IA2CnE;;OAEG;IACH,SAAS,CAAC,kBAAkB,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBACpB,IAAI;;IAgC7F,SAAS,CAAC,eAAe,CACrB,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,cAAc,GACzB;QACC,QAAQ,EAAE,cAAc,CAAC;QACzB,QAAQ,EAAE,QAAQ,CAAC;KACtB;IAsCD;;OAEG;IACH,SAAS,CAAC,yBAAyB,CAAC,mBAAmB,EAAE,CAAC,MAAM,GAAG,WAAW,GAAG,YAAY,CAAC,EAAE;IAgBhG;;OAEG;IACH,SAAS,CAAC,qBAAqB,CAAC,OAAO,CAAC,EAAE,OAAO;IAKjD,OAAO,CAAC,oBAAoB;IAkB5B;;OAEG;IACH,OAAO,CAAC,iBAAiB,CAuBvB;CACL;AAED,UAAU,sBAAsB;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,WAAW,CAAC;CAC3B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,gBAAgB,CAC5B,OAAO,SAAS,mBAAmB,GAAG,mBAAmB,EACzD,QAAQ,SAAS,UAAU,GAAG,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAC1E,MAAM,CAAC,EAAE,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,mDAEzC"}
1
+ {"version":3,"file":"http-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/http-crawler.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAER,mBAAmB,EACnB,eAAe,EACf,YAAY,EACZ,sBAAsB,EACtB,OAAO,IAAI,cAAc,EACzB,cAAc,EACd,sBAAsB,EACtB,YAAY,EACZ,OAAO,EACV,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACH,YAAY,EAEZ,aAAa,EACb,eAAe,EAOlB,MAAM,gBAAgB,CAAC;AACxB,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5D,OAAO,EAAE,KAAK,WAAW,EAAuB,MAAM,gBAAgB,CAAC;AAEvE,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAK9D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAwB3C,MAAM,MAAM,gBAAgB,CACxB,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,IAChC,YAAY,CAAC,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE1D,MAAM,WAAW,kBAAkB,CAC/B,OAAO,SAAS,2BAA2B,GAAG,2BAA2B,EACzE,gBAAgB,GAAG,UAAU,CAAC,KAAK,CAAC,EACpC,eAAe,SAAS,OAAO,GAAG,OAAO,GAAG,gBAAgB,CAC9D,SAAQ,mBAAmB,CAAC,OAAO,EAAE,gBAAgB,EAAE,eAAe,CAAC;IACrE;;OAEG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;OAEG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;;;;;;;;;;;;;;OAeG;IACH,kBAAkB,CAAC,EAAE,gBAAgB,CAAC,eAAe,CAAC,EAAE,CAAC;IAEzD;;;;;;;;;;;OAWG;IACH,mBAAmB,CAAC,EAAE,CAAC,CAAC,eAAe,EAAE,0BAA0B,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;IAE3F;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE/B;;;;;;;;;;OAUG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAEjC;;;;;;;;OAQG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;;;OAKG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAC;IAEnC;;;OAGG;IACH,0BAA0B,CAAC,EAAE,MAAM,EAAE,CAAC;IAEtC;;;OAGG;IACH,8BAA8B,CAAC,EAAE,MAAM,EAAE,CAAC;CAC7C;AAED;;GAEG;AACH,MAAM,MAAM,gBAAgB,CAAC,OAAO,IAAI,CAAC,eAAe,EAAE,OAAO,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAEtF,MAAM,MAAM,QAAQ,CAChB,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,IAChC,gBAAgB,CAAC,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE9D,UAAU,0BAA0B,CAChC,QAAQ,SAAS,UAAU,GAAG,GAAG,CACnC,SAAQ,eAAe,CAAC,QAAQ,CAAC;IAC/B;;OAEG;IACH,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,CAAC;IAEjD;;OAEG;IACH,QAAQ,EAAE,QAAQ,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,2BAA2B,CACxC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,CAClC,SAAQ,0BAA0B,CAAC,QAAQ,CAAC;IAC1C;;;;;OAKG;IACH,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;IAEtB;;OAEG;IACH,IAAI,EAAE,QAAQ,CAAC;IAEf;;OAEG;IACH,WAAW,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,cAAc,CAAA;KAAE,CAAC;IAExD;;;;;;;;;;;OAWG;IACH,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErE;;;;;;;;;;;OAWG;IACH,gBAAgB,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;CACjF;AAED,MAAM,WAAW,mBAAmB,CAAC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,QAAQ,SAAS,SAAS,GAAG,GAAG,CACpG,SAAQ,2BAA2B,CAAC,QAAQ,EAAE,QAAQ,CAAC;CAAG;AAE9D,MAAM,MAAM,kBAAkB,CAC1B,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,IAChC,cAAc,CAAC,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkEG;AACH,qBAAa,WAAW,CACpB,OAAO,SAAS,2BAA2B,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,2BAA2B,EACnF,gBAAgB,GAAG,UAAU,CAAC,KAAK,CAAC,EACpC,eAAe,SAAS,OAAO,GAAG,OAAO,GAAG,gBAAgB,CAC9D,SAAQ,YAAY,CAAC,OAAO,EAAE,gBAAgB,EAAE,eAAe,CAAC;aAmCxC,MAAM;IAlC5B,SAAS,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,eAAe,CAAC,EAAE,CAAC;IAClE,SAAS,CAAC,mBAAmB,EAAE,CAAC,CAAC,eAAe,EAAE,0BAA0B,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;IACpG,SAAS,CAAC,wBAAwB,EAAE,OAAO,CAAC;IAC5C,SAAS,CAAC,uBAAuB,EAAE,MAAM,CAAC;IAC1C,SAAS,CAAC,eAAe,EAAE,OAAO,CAAC;IACnC,SAAS,CAAC,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAC3C,SAAS,CAAC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IACzC,SAAS,CAAC,8BAA8B,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACtD,SAAS,CAAC,0BAA0B,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAClD,SAAS,CAAC,QAAQ,CAAC,kBAAkB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAEnD,iBAA0B,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAepC;IAEF;;OAEG;gBAEC,OAAO,GAAE,kBAAkB,CAAC,OAAO,EAAE,gBAAgB,EAAE,eAAe,CAAC,GACnE,sBAAsB,CAAC,2BAA2B,EAAE,OAAO,CAAa,EAC1D,MAAM,gBAAkC;IAkE9D,SAAS,CAAC,oBAAoB,IAAI,eAAe,CAAC,eAAe,EAAE,2BAA2B,CAAC;YASjF,eAAe;YAkDf,mBAAmB;YAuEnB,6BAA6B;cAQ3B,gBAAgB,CAAC,eAAe,EAAE,2BAA2B,GAAG,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC;IAyBvG;;;OAGG;IACH,SAAS,CAAC,aAAa,CACnB,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,eAAe,EACrC,cAAc,EAAE,MAAM,EACtB,eAAe,EAAE,MAAM,GACxB,MAAM;IAQT;;;;OAIG;cACa,gBAAgB,CAAC,EAC7B,OAAO,EACP,OAAO,EACP,QAAQ,EACR,YAAY,GACf,EAAE,sBAAsB,GAAG,OAAO,CAAC,QAAQ,CAAC;IAwB7C;;OAEG;cACa,cAAc,CAAC,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,QAAQ;;;;;;;;;;;;;;;IA2C1E;;OAEG;IACH,SAAS,CAAC,kBAAkB,CAAC,OAAO,EAAE,cAAc,EAAE,OAAO,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,MAAM;;;;;;;;;;;;;cAY/D,MAAM,GAAG,SAAS;;IAmB7C,SAAS,CAAC,eAAe,CACrB,OAAO,EAAE,cAAc,EACvB,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,cAAc,GACzB;QACC,QAAQ,EAAE,cAAc,CAAC;QACzB,QAAQ,EAAE,QAAQ,CAAC;KACtB;IAsCD;;OAEG;IACH,SAAS,CAAC,yBAAyB,CAAC,mBAAmB,EAAE,CAAC,MAAM,GAAG,WAAW,GAAG,YAAY,CAAC,EAAE;IAgBhG;;OAEG;IACH,SAAS,CAAC,qBAAqB,CAAC,OAAO,CAAC,EAAE,OAAO;IAKjD,OAAO,CAAC,oBAAoB;IAkB5B;;OAEG;IACH,OAAO,CAAC,iBAAiB,CAsCvB;CACL;AAED,UAAU,sBAAsB;IAC5B,OAAO,EAAE,cAAc,CAAC;IACxB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,gBAAgB,CAC5B,OAAO,SAAS,mBAAmB,GAAG,mBAAmB,EACzD,QAAQ,SAAS,UAAU,GAAG,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAC1E,MAAM,CAAC,EAAE,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,mDAEzC"}
package/internals/http-crawler.js CHANGED
@@ -45,11 +45,11 @@ const HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
45
45
  *
46
46
  * The crawler finishes when there are no more {@link Request} objects to crawl.
47
47
  *
48
- * We can use the `preNavigationHooks` to adjust `gotOptions`:
48
+ * We can use the `preNavigationHooks` to adjust the crawling context before the request is made:
49
49
  *
50
50
  * ```javascript
51
51
  * preNavigationHooks: [
52
- * (crawlingContext, gotOptions) => {
52
+ * (crawlingContext) => {
53
53
  * // ...
54
54
  * },
55
55
  * ]
@@ -184,17 +184,16 @@ export class HttpCrawler extends BasicCrawler {
184
184
  },
185
185
  };
186
186
  }
187
- const gotOptions = {};
188
187
  const preNavigationHooksCookies = this._getCookieHeaderFromRequest(request);
189
188
  request.state = RequestState.BEFORE_NAV;
190
189
  // Execute pre navigation hooks before applying session pool cookies,
191
190
  // as they may also set cookies in the session
192
- await this._executeHooks(this.preNavigationHooks, crawlingContext, gotOptions);
191
+ await this._executeHooks(this.preNavigationHooks, crawlingContext);
193
192
  tryCancel();
194
193
  const postNavigationHooksCookies = this._getCookieHeaderFromRequest(request);
195
- this._applyCookies(crawlingContext, gotOptions, preNavigationHooksCookies, postNavigationHooksCookies);
194
+ const cookieString = this._applyCookies(crawlingContext, preNavigationHooksCookies, postNavigationHooksCookies);
196
195
  const proxyUrl = crawlingContext.proxyInfo?.url;
197
- const httpResponse = await addTimeoutToPromise(async () => this._requestFunction({ request, session, proxyUrl, gotOptions }), this.navigationTimeoutMillis, `request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
196
+ const httpResponse = await addTimeoutToPromise(async () => this._requestFunction({ request, session, proxyUrl, cookieString }), this.navigationTimeoutMillis, `request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
198
197
  tryCancel();
199
198
  request.loadedUrl = httpResponse?.url;
200
199
  request.state = RequestState.AFTER_NAV;
@@ -286,58 +285,27 @@ export class HttpCrawler extends BasicCrawler {
286
285
  return false;
287
286
  }
288
287
  /**
289
- * Sets the cookie header to `gotOptions` based on the provided request and session headers, as well as any changes that occurred due to hooks.
288
+ * Returns the `Cookie` header value based on the current context and
289
+ * any changes that occurred in the navigation hooks.
290
290
  */
291
- _applyCookies({ session, request }, gotOptions, preHookCookies, postHookCookies) {
291
+ _applyCookies({ session, request }, preHookCookies, postHookCookies) {
292
292
  const sessionCookie = session?.getCookieString(request.url) ?? '';
293
- let alteredGotOptionsCookies = gotOptions.headers?.Cookie || gotOptions.headers?.cookie || '';
294
- if (gotOptions.headers?.Cookie && gotOptions.headers?.cookie) {
295
- const { Cookie: upperCaseHeader, cookie: lowerCaseHeader } = gotOptions.headers;
296
- this.log.warning(`Encountered mixed casing for the cookie headers in the got options for request ${request.url} (${request.id}). Their values will be merged`);
297
- const sourceCookies = [];
298
- if (Array.isArray(lowerCaseHeader)) {
299
- sourceCookies.push(...lowerCaseHeader);
300
- }
301
- else {
302
- sourceCookies.push(lowerCaseHeader);
303
- }
304
- if (Array.isArray(upperCaseHeader)) {
305
- sourceCookies.push(...upperCaseHeader);
306
- }
307
- else {
308
- sourceCookies.push(upperCaseHeader);
309
- }
310
- alteredGotOptionsCookies = mergeCookies(request.url, sourceCookies);
311
- }
312
- const sourceCookies = [sessionCookie, preHookCookies];
313
- if (Array.isArray(alteredGotOptionsCookies)) {
314
- sourceCookies.push(...alteredGotOptionsCookies);
315
- }
316
- else {
317
- sourceCookies.push(alteredGotOptionsCookies);
318
- }
319
- sourceCookies.push(postHookCookies);
320
- const mergedCookie = mergeCookies(request.url, sourceCookies);
321
- gotOptions.headers ??= {};
322
- Reflect.deleteProperty(gotOptions.headers, 'Cookie');
323
- Reflect.deleteProperty(gotOptions.headers, 'cookie');
324
- if (mergedCookie !== '') {
325
- gotOptions.headers.Cookie = mergedCookie;
326
- }
293
+ const sourceCookies = [sessionCookie, preHookCookies, postHookCookies];
294
+ return mergeCookies(request.url, sourceCookies);
327
295
  }
328
296
  /**
329
297
  * Function to make the HTTP request. It performs optimizations
330
298
  * on the request such as only downloading the request body if the
331
299
  * received content type matches text/html, application/xml, application/xhtml+xml.
332
300
  */
333
- async _requestFunction({ request, session, proxyUrl, gotOptions, }) {
301
+ async _requestFunction({ request, session, proxyUrl, cookieString, }) {
334
302
  if (!TimeoutError) {
335
303
  // @ts-ignore
336
304
  ({ TimeoutError } = await import('got-scraping'));
337
305
  }
338
- const opts = this._getRequestOptions(request, session, proxyUrl, gotOptions);
306
+ const opts = this._getRequestOptions(request, session, proxyUrl);
339
307
  try {
340
- return await this._requestAsBrowser(opts, session);
308
+ return await this._requestAsBrowser(opts, session, cookieString);
341
309
  }
342
310
  catch (e) {
343
311
  if (e instanceof TimeoutError) {
@@ -397,7 +365,7 @@ export class HttpCrawler extends BasicCrawler {
397
365
  /**
398
366
  * Combines the provided `requestOptions` with mandatory (non-overridable) values.
399
367
  */
400
- _getRequestOptions(request, session, proxyUrl, gotOptions) {
368
+ _getRequestOptions(request, session, proxyUrl) {
401
369
  const requestOptions = {
402
370
  url: request.url,
403
371
  method: request.method,
@@ -405,13 +373,11 @@ export class HttpCrawler extends BasicCrawler {
405
373
  timeout: { request: this.navigationTimeoutMillis },
406
374
  cookieJar: this.persistCookiesPerSession ? session?.cookieJar : undefined,
407
375
  sessionToken: session,
408
- ...gotOptions,
409
- headers: { ...request.headers, ...gotOptions?.headers },
376
+ headers: request.headers,
410
377
  https: {
411
- ...gotOptions?.https,
412
378
  rejectUnauthorized: !this.ignoreSslErrors,
413
379
  },
414
- isStream: true,
380
+ body: undefined,
415
381
  };
416
382
  // Delete any possible lowercased header for cookie as they are merged in _applyCookies under the uppercase Cookie header
417
383
  Reflect.deleteProperty(requestOptions.headers, 'cookie');
@@ -498,19 +464,35 @@ export class HttpCrawler extends BasicCrawler {
498
464
  /**
499
465
  * @internal wraps public utility for mocking purposes
500
466
  */
501
- _requestAsBrowser = async (options, session) => {
502
- const response = await this.httpClient.stream(processHttpRequestOptions({
467
+ _requestAsBrowser = async (options, session, cookieString) => {
468
+ const opts = processHttpRequestOptions({
503
469
  ...options,
504
470
  cookieJar: options.cookieJar,
505
471
  responseType: 'text',
506
- }), (redirectResponse, updatedRequest) => {
507
- if (this.persistCookiesPerSession) {
508
- session.setCookiesFromResponse(redirectResponse);
509
- const cookieString = session.getCookieString(updatedRequest.url.toString());
510
- if (cookieString !== '') {
511
- updatedRequest.headers.Cookie = cookieString;
472
+ });
473
+ if (cookieString) {
474
+ opts.headers?.delete('Cookie');
475
+ opts.headers?.delete('cookie');
476
+ opts.headers?.set('Cookie', cookieString);
477
+ }
478
+ const response = await this.httpClient.stream(new Request(opts.url, {
479
+ body: opts.body ? Readable.toWeb(opts.body) : undefined,
480
+ headers: new Headers(opts.headers),
481
+ method: opts.method,
482
+ // Node-specific option to make the request body work with streams
483
+ duplex: 'half',
484
+ }), {
485
+ session,
486
+ timeout: opts.timeout,
487
+ onRedirect: (redirectResponse, updatedRequest) => {
488
+ if (this.persistCookiesPerSession) {
489
+ session.setCookiesFromResponse(redirectResponse);
490
+ const cookieStringRedirected = session.getCookieString(updatedRequest.url.toString());
491
+ if (cookieStringRedirected !== '') {
492
+ updatedRequest.headers.set('Cookie', cookieStringRedirected);
493
+ }
512
494
  }
513
- }
495
+ },
514
496
  });
515
497
  return response;
516
498
  };
package/internals/http-crawler.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"http-crawler.js","sourceRoot":"","sources":["../../src/internals/http-crawler.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,IAAI,MAAM,WAAW,CAAC;AAc7B,OAAO,EACH,YAAY,EACZ,oBAAoB,EACpB,aAAa,EACb,eAAe,EACf,YAAY,EACZ,yBAAyB,EACzB,YAAY,EACZ,eAAe,EACf,MAAM,EACN,YAAY,GACf,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EAAoB,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AACvE,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,OAAO,iBAAiB,MAAM,cAAc,CAAC;AAE7C,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,EAAE,MAAM,IAAI,CAAC;AAGpB,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAEhE,OAAO,EAAE,4BAA4B,EAAE,MAAM,YAAY,CAAC;AAE1D,IAAI,YAAsC,CAAC;AAW3C;;GAEG;AACH,MAAM,uBAAuB,GAAG,CAAC,WAAW,EAAE,UAAU,EAAE,uBAAuB,EAAE,iBAAiB,CAAC,CAAC;AACtG,MAAM,0BAA0B,GAAG,kBAAkB,CAAC;AACtD,MAAM,sCAAsC,GAA0B;IAClE,kBAAkB,EAAE,EAAE;IACtB,kBAAkB,EAAE;QAChB,6BAA6B,EAAE,CAAC;QAChC,gBAAgB,EAAE,GAAG;KACxB;IACD,mBAAmB,EAAE;QACjB,2BAA2B,EAAE,GAAG;KACnC;CACJ,CAAC;AA+LF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkEG;AACH,MAAM,OAAO,WAIX,SAAQ,YAAwD;IAmCxC;IAlCZ,kBAAkB,CAAsC;IACxD,mBAAmB,CAAuE;IAC1F,wBAAwB,CAAU;IAClC,uBAAuB,CAAS;IAChC,eAAe,CAAU;IACzB,uBAAuB,CAAU;IACjC,qBAAqB,CAAU;IAC/B,8BAA8B,CAAc;IAC5C,0BAA0B,CAAc;IAC/B,kBAAkB,CAAc;IAEzC,MAAM,CAAU,YAAY,GAAG;QACrC,GAAG,YAAY,CAAC,YAAY;QAE5B,qBAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QACpC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;QACxD,uBAAuB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC3C,qBAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,wBAAwB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QAE7C,8BAA8B,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;QACnE,0BAA0B,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;QAE/D,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK;QACrC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK;KACzC,CAAC;IAEF;;OAEG;IACH,YACI,UACmE,EAAS,EAC1D,SAAS,aAAa,CAAC,eAAe,EAAE;QAE1D,EAAE,CAAC,OAAO,EAAE,oBAAoB,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC;QAElF,MAAM,EACF,qBAAqB,GAAG,EAAE,EAC1B,eAAe,GAAG,IAAI,EACtB,mBAAmB,GAAG,EAAE,EACxB,uBAAuB,EACvB,qBAAqB,EACrB,wBAAwB,EACxB,kBAAkB,GAAG,EAAE,EACvB,mBAAmB,GAAG,EAAE,EACxB,8BAA8B,GAAG,EAAE,EACnC,0BAA0B,GAAG,EAAE;QAE/B,eAAe;QACf,qBAAqB,GAAG,sCAAsC,EAC9D,sBAAsB,EACtB,GAAG,mBAAmB,EACzB,GAAG,OAAO,CAAC;QAEZ,KAAK,CACD;YACI,GAAG,mBAAmB;YACtB,qBAAqB;YACrB,sBAAsB,EAClB,sBAAsB;gBACtB,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,oBAAoB,EAA+C,CAAC;SACvF,EACD,MAAM,CACT,CAAC;QA/BgB,WAAM,GAAN,MAAM,CAAkC;QAiC1D,uEAAuE;QACvE,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,wBAAwB,EAAE,CAAC;YACnD,MAAM,IAAI,KAAK,CAAC,iFAAiF,CAAC,CAAC;QACvG,CAAC;QAED,IAAI,CAAC,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,uBAAuB,EAAE,0BAA0B,CAAC,CAAC,CAAC;QAC5F,IAAI,mBAAmB,CAAC,MAAM;YAAE,IAAI,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC;QAEpF,IAAI,uBAAuB,IAAI,qBAAqB,EAAE,CAAC;YACnD,IAAI,CAAC,GAAG,CAAC,OAAO,CACZ,sGAAsG,CACzG,CAAC;QACN,CAAC;QAED,IAAI,CAAC,uBAAuB,GAAG,qBAAqB,GAAG,IAAI,CAAC;QAC5D,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,uBAAuB,GAAG,uBAAuB,CAAC;QACvD,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,CAAC;QACnD,IAAI,CAAC,8BAA8B,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,8BAA8B,CAAC,CAAC,CAAC;QACnF,IAAI,CAAC,0BAA0B,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,0BAA0B,CAAC,CAAC,CAAC;QAC3E,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,mBAAmB,GAAG;YACvB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,oBAAoB,CAAC,OAAO,EAAE,QAAS,CAAC;YACxE,GAAG,mBAAmB;SACzB,CAAC;QAEF,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,CAAC,wBAAwB,GAAG,wBAAwB,IAAI,IAAI,CAAC;QACrE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,wBAAwB,GAAG,KAAK,CAAC;QAC1C,CAAC;IACL,CAAC;IAES,oBAAoB;QAC1B,OAAO,eAAe,CAAC,MAAM,EAAmB;aAC3C,OAAO,CAAC;YACL,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC;SAC1C,CAAC;aACD,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;aACxD,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,6BAA6B,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC5E,CAAC;IAEO,KAAK,CAAC,eAAe,CACzB,eAAgC;QAEhC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAE7C,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YACzB,OAAO;gBACH,OAAO,EAAE,IAAI,KAAK,CAAC,OAAO,EAAE;oBACxB,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ;wBAC9B,IAAI,YAAY,KAAK,WAAW,EAAE,CAAC;4BAC/B,MAAM,IAAI,KAAK,CACX,+EAA+E,CAClF,CAAC;wBACN,CAAC;wBACD,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ,CAAC,CAAC;oBACvD,CAAC;iBACJ,CAA2B;gBAC5B,IAAI,QAAQ;oBACR,MAAM,IAAI,KAAK,CAAC,sEAAsE,CAAC,CAAC;gBAC5F,CAAC;aACJ,CAAC;QACN,CAAC;QAED,MAAM,UAAU,GAAG,EAAiB,CAAC;QACrC,MAAM,yBAAyB,GAAG,IAAI,CAAC,2BAA2B,CAAC,OAAO,CAAC,CAAC;QAE5E,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,UAAU,CAAC;QACxC,qEAAqE;QACrE,8CAA8C;QAC9C,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,kBAAkB,EAAE,eAAe,EAAE,UAAU,CAAC,CAAC;QAC/E,SAAS,EAAE,CAAC;QAEZ,MAAM,0BAA0B,GAAG,IAAI,CAAC,2BAA2B,CAAC,OAAO,CAAC,CAAC;QAE7E,IAAI,CAAC,aAAa,CAAC,eAAe,EAAE,UAAU,EAAE,yBAAyB,EAAE,0BAA0B,CAAC,CAAC;QAEvG,MAAM,QAAQ,GAAG,eAAe,CAAC,SAAS,EAAE,GAAG,CAAC;QAEhD,MAAM,YAAY,GAAG,MAAM,mBAAmB,CAC1C,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAC,EAC7E,IAAI,CAAC,uBAAuB,EAC5B,2BAA2B,IAAI,CAAC,uBAAuB,GAAG,IAAI,WAAW,CAC5E,CAAC;QACF,SAAS,EAAE,CAAC;QAEZ,OAAO,CAAC,SAAS,GAAG,YAAY,EAAE,GAAG,CAAC;QACtC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,SAAS,CAAC;QAEvC,OAAO,EAAE,OAAO,EAAE,OAAiC,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC;IAClF,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAC7B,eAA2C;QAI3C,IAAI,eAAe,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YACzC,OAAO;gBACH,IAAI,WAAW;oBACX,MAAM,IAAI,KAAK,CAAC,yEAAyE,CAAC,CAAC;gBAC/F,CAAC;gBACD,IAAI,IAAI;oBACJ,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;gBACxF,CAAC;gBACD,IAAI,IAAI;oBACJ,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;gBACxF,CAAC;gBACD,IAAI,eAAe;oBACf,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAC;gBACjG,CAAC;gBACD,IAAI,gBAAgB;oBAChB,MAAM,IAAI,KAAK,CAAC,4EAA4E,CAAC,CAAC;gBAClG,CAAC;aACJ,CAAC;QACN,CAAC;QAED,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,mBAAmB,EAAE,eAAe,CAAC,CAAC;QACpE,SAAS,EAAE,CAAC;QAEZ,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,OAAO,EAAE,eAAe,CAAC,QAAQ,CAAC,CAAC;QAC5F,SAAS,EAAE,CAAC;QACZ,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAS,CAAC;QAClC,MAAM,WAAW,GAAG,MAAM,CAAC,WAAY,CAAC;QAExC,MAAM,eAAe,GAAG,KAAK,EAAE,QAAgB,EAAE,UAAmB,EAAE,EAAE;YACpE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;YAEhD,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACjC,MAAM,IAAI,KAAK,CAAC,aAAa,QAAQ,cAAc,CAAC,CAAC;YACzD,CAAC;QACL,CAAC,CAAC;QACF,MAAM,gBAAgB,GAAG,KAAK,EAAE,QAAiB,EAAE,SAAkB,EAAE,EAAE;YACrE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;YAEhD,IAAI,QAAQ,EAAE,CAAC;gBACX,MAAO,eAA+C,CAAC,eAAe,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;YAChG,CAAC;YAED,OAAO,CAAC,CAAC;QACb,CAAC,CAAC;QAEF,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,CAAC,sBAAsB,CAAC,eAAe,CAAC,OAAQ,EAAE,QAAQ,CAAC,MAAO,CAAC,CAAC;QAC5E,CAAC;QAED,IAAI,IAAI,CAAC,wBAAwB,EAAE,CAAC;YAChC,eAAe,CAAC,OAAQ,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC;QAC9D,CAAC;QAED,OAAO;YACH,IAAI,IAAI;gBACJ,IAAI,WAAW,CAAC,IAAI,KAAK,0BAA0B;oBAAE,OAAO,IAAI,CAAC;gBACjE,MAAM,UAAU,GAAG,MAAM,CAAC,IAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;gBAC/D,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAClC,CAAC;YACD,eAAe;YACf,gBAAgB;YAChB,WAAW;YACX,IAAI,EAAE,MAAM,CAAC,IAAI;SACpB,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,6BAA6B,CAAC,eAA4C;QACpF,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC;YAC3D,IAAI,KAAK;gBAAE,MAAM,IAAI,YAAY,CAAC,KAAK,CAAC,CAAC;QAC7C,CAAC;QACD,OAAO,EAAE,CAAC;IACd,CAAC;IAES,KAAK,CAAC,gBAAgB,CAAC,eAA4C;QACzE,IAAI,uBAAuB,CAAC,QAAQ,CAAC,eAAe,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;YACrE,MAAM,CAAC,GAAG,MAAM,eAAe,CAAC,gBAAgB,EAAE,CAAC;YAEnD,MAAM,cAAc,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAExF,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,OAAO,oBAAoB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3D,CAAC;QACL,CAAC;QAED,MAAM,kBAAkB;QACpB,wCAAwC;QACxC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,oBAAoB,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC;YACtD,CAAC,CAAC,wCAAwC;gBACxC,IAAI,CAAC,WAAY,CAAC,oBAAoB,CAAC;YACzC,CAAC,CAAC,oBAAoB,CAAC;QAE/B,IAAI,kBAAkB,CAAC,QAAQ,CAAC,eAAe,CAAC,QAAQ,CAAC,MAAO,CAAC,EAAE,CAAC;YAChE,OAAO,0BAA0B,eAAe,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;QACvE,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;OAEG;IACO,aAAa,CACnB,EAAE,OAAO,EAAE,OAAO,EAAmB,EACrC,UAAuB,EACvB,cAAsB,EACtB,eAAuB;QAEvB,MAAM,aAAa,GAAG,OAAO,EAAE,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QAClE,IAAI,wBAAwB,GAAG,UAAU,CAAC,OAAO,EAAE,MAAM,IAAI,UAAU,CAAC,OAAO,EAAE,MAAM,IAAI,EAAE,CAAC;QAE9F,IAAI,UAAU,CAAC,OAAO,EAAE,MAAM,IAAI,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YAC3D,MAAM,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,EAAE,eAAe,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC;YAEhF,IAAI,CAAC,GAAG,CAAC,OAAO,CACZ,kFAAkF,OAAO,CAAC,GAAG,KAAK,OAAO,CAAC,EAAE,gCAAgC,CAC/I,CAAC;YAEF,MAAM,aAAa,GAAG,EAAE,CAAC;YAEzB,IAAI,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;gBACjC,aAAa,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,CAAC;YAC3C,CAAC;iBAAM,CAAC;gBACJ,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YACxC,CAAC;YAED,IAAI,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;gBACjC,aAAa,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,CAAC;YAC3C,CAAC;iBAAM,CAAC;gBACJ,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YACxC,CAAC;YAED,wBAAwB,GAAG,YAAY,CAAC,OAAO,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;QACxE,CAAC;QAED,MAAM,aAAa,GAAG,CAAC,aAAa,EAAE,cAAc,CAAC,CAAC;QAEtD,IAAI,KAAK,CAAC,OAAO,CAAC,wBAAwB,CAAC,EAAE,CAAC;YAC1C,aAAa,CAAC,IAAI,CAAC,GAAG,wBAAwB,CAAC,CAAC;QACpD,CAAC;aAAM,CAAC;YACJ,aAAa,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;QACjD,CAAC;QAED,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAEpC,MAAM,YAAY,GAAG,YAAY,CAAC,OAAO,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;QAE9D,UAAU,CAAC,OAAO,KAAK,EAAE,CAAC;QAC1B,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QACrD,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QAErD,IAAI,YAAY,KAAK,EAAE,EAAE,CAAC;YACtB,UAAU,CAAC,OAAO,CAAC,MAAM,GAAG,YAAY,CAAC;QAC7C,CAAC;IACL,CAAC;IAED;;;;OAIG;IACO,KAAK,CAAC,gBAAgB,CAAC,EAC7B,OAAO,EACP,OAAO,EACP,QAAQ,EACR,UAAU,GACW;QACrB,IAAI,CAAC,YAAY,EAAE,CAAC;YAChB,aAAa;YACb,CAAC,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC;QACtD,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;QAE7E,IAAI,CAAC;YACD,OAAO,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACvD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,YAAY,YAAY,EAAE,CAAC;gBAC5B,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;gBACpC,OAAO,IAAI,QAAQ,EAAE,CAAC,CAAC,iEAAiE;YAC5F,CAAC;YAED,IAAI,IAAI,CAAC,YAAY,CAAC,CAAU,CAAC,EAAE,CAAC;gBAChC,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAU,CAAW,CAAC,CAAC;YAC5E,CAAC;iBAAM,CAAC;gBACJ,MAAM,CAAC,CAAC;YACZ,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,cAAc,CAAC,OAAgB,EAAE,QAAkB;QAC/D,MAAM,EAAE,MAAM,EAAE,GAAG,QAAQ,CAAC;QAC5B,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,4BAA4B,CAAC,QAAQ,CAAC,CAAC;QACjE,MAAM,EAAE,QAAQ,EAAE,iBAAiB,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;QACnG,MAAM,WAAW,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QAEvC,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;YACjC,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;QAC1C,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,CAAC,0BAA0B,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACjE,MAAM,YAAY,GAAG,IAAI,CAAC,8BAA8B,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAErE,IAAI,CAAC,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,YAAY,EAAE,CAAC;YACnD,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,IAAI,EAAE,CAAC,CAAC,oGAAoG;YAEjJ,2DAA2D;YAC3D,gDAAgD;YAChD,IAAI,IAAI,KAAK,0BAA0B,EAAE,CAAC;gBACtC,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACvC,IAAI,EAAE,OAAO,EAAE,GAAG,aAAa,CAAC;gBAChC,IAAI,CAAC,OAAO;oBAAE,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,cAAc,EAAE,EAAE,EAAE,CAAC,CAAC;gBACtF,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,MAAM,OAAO,EAAE,CAAC,CAAC;YAC9C,CAAC;YAED,IAAI,YAAY,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,uCAAuC,CAAC,CAAC;YACtE,CAAC;YAED,8EAA8E;YAC9E,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,6BAA6B,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAChF,CAAC;aAAM,IAAI,uBAAuB,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAChD,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;QAClE,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;YACjD,OAAO;gBACH,IAAI;gBACJ,QAAQ;gBACR,WAAW;aACd,CAAC;QACN,CAAC;IACL,CAAC;IAED;;OAEG;IACO,kBAAkB,CAAC,OAAgB,EAAE,OAAiB,EAAE,QAAiB,EAAE,UAAwB;QACzG,MAAM,cAAc,GAA0E;YAC1F,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,MAAM,EAAE,OAAO,CAAC,MAAgB;YAChC,QAAQ;YACR,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,uBAAuB,EAAE;YAClD,SAAS,EAAE,IAAI,CAAC,wBAAwB,CAAC,CAAC,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,SAAS;YACzE,YAAY,EAAE,OAAO;YACrB,GAAG,UAAU;YACb,OAAO,EAAE,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,GAAG,UAAU,EAAE,OAAO,EAAE;YACvD,KAAK,EAAE;gBACH,GAAG,UAAU,EAAE,KAAK;gBACpB,kBAAkB,EAAE,CAAC,IAAI,CAAC,eAAe;aAC5C;YACD,QAAQ,EAAE,IAAI;SACjB,CAAC;QAEF,yHAAyH;QACzH,OAAO,CAAC,cAAc,CAAC,cAAc,CAAC,OAAQ,EAAE,QAAQ,CAAC,CAAC;QAE1D,4CAA4C;QAC5C,IAAI,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,CAAC;YACtC,cAAc,CAAC,KAAK,GAAG;gBACnB,GAAG,cAAc,CAAC,KAAK;gBACvB,kBAAkB,EAAE,KAAK;aAC5B,CAAC;QACN,CAAC;QAED,IAAI,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,cAAc,CAAC,IAAI,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;QAEvF,OAAO,cAAc,CAAC;IAC1B,CAAC;IAES,eAAe,CACrB,OAAgB,EAChB,QAAkB,EAClB,QAAwB;QAKxB,IAAI,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAC7B,QAAQ,GAAG,IAAI,CAAC,qBAAuC,CAAC;QAC5D,CAAC;aAAM,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,uBAAuB,EAAE,CAAC;YACnD,QAAQ,GAAG,IAAI,CAAC,uBAAyC,CAAC;QAC9D,CAAC;QAED,sDAAsD;QACtD,MAAM,IAAI,GAAG,MAAM,CAAC;QACpB,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAEnD,2DAA2D;QAC3D,+CAA+C;QAC/C,IAAI,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;QAE/D,+DAA+D;QAC/D,IAAI,KAAK,CAAC,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC;YACjC,MAAM,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YACxG,MAAM,aAAa,GAAG,QAAQ,CAAC,IAAI;gBAC/B,CAAC,CAAC,QAAQ,CAAC,KAAK,CACV,QAAQ,CAAC,IAAI,CACT,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAW,CAAC;qBACjC,IAAI,CAAC,YAAY,CAAC;qBAClB,IAAI,CAAC,YAAY,CAAC,CAC1B,CACJ;gBACH,CAAC,CAAC,IAAI,CAAC;YAEX,OAAO;gBACH,QAAQ,EAAE,IAAI,eAAe,CAAC,aAAoB,EAAE,QAAQ,CAAC;gBAC7D,QAAQ,EAAE,IAAI;aACjB,CAAC;QACN,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,YAAY,OAAO,CAAC,GAAG,8CAA8C,QAAQ,EAAE,CAAC,CAAC;IACrG,CAAC;IAED;;OAEG;IACO,yBAAyB,CAAC,mBAA4D;QAC5F,KAAK,MAAM,QAAQ,IAAI,mBAAmB,EAAE,CAAC;YACzC,IAAI,QAAQ,KAAK,KAAK,EAAE,CAAC;gBACrB,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBACtC,SAAS;YACb,CAAC;YAED,IAAI,CAAC;gBACD,MAAM,UAAU,GAAG,iBAAiB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBACrD,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;YACjD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,IAAI,KAAK,CAAC,2BAA2B,QAAQ,sCAAsC,CAAC,CAAC;YAC/F,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACO,qBAAqB,CAAC,OAAiB;QAC7C,OAAO,EAAE,OAAO,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,uBAAuB,GAAG,IAAI,WAAW,CAAC,CAAC;IAC/F,CAAC;IAEO,oBAAoB,CAAC,OAAgB,EAAE,QAAkB;QAC7D,MAAM,EAAE,MAAM,EAAE,GAAG,QAAQ,CAAC;QAC5B,MAAM,EAAE,IAAI,EAAE,GAAG,4BAA4B,CAAC,QAAQ,CAAC,CAAC;QAExD,sEAAsE;QACtE,MAAM,kBAAkB,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1F,wDAAwD;QACxD,MAAM,sBAAsB,GAAG,MAAM,IAAI,GAAG,IAAI,kBAAkB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAEpF,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;YACvG,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;YACvB,MAAM,IAAI,KAAK,CACX,YAAY,OAAO,CAAC,GAAG,wBAAwB,IAAI,IAAI;gBACnD,YAAY,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,kCAAkC,CACnG,CAAC;QACN,CAAC;IACL,CAAC;IAED;;OAEG;IACK,iBAAiB,GAAG,KAAK,EAC7B,OAA4D,EAC5D,OAAiB,EACnB,EAAE;QACA,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CACzC,yBAAyB,CAAC;YACtB,GAAI,OAAe;YACnB,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,YAAY,EAAE,MAAM;SACvB,CAAC,EACF,CAAC,gBAAgB,EAAE,cAAc,EAAE,EAAE;YACjC,IAAI,IAAI,CAAC,wBAAwB,EAAE,CAAC;gBAChC,OAAQ,CAAC,sBAAsB,CAAC,gBAAgB,CAAC,CAAC;gBAElD,MAAM,YAAY,GAAG,OAAQ,CAAC,eAAe,CAAC,cAAc,CAAC,GAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAC9E,IAAI,YAAY,KAAK,EAAE,EAAE,CAAC;oBACtB,cAAc,CAAC,OAAO,CAAC,MAAM,GAAG,YAAY,CAAC;gBACjD,CAAC;YACL,CAAC;QACL,CAAC,CACJ,CAAC;QAEF,OAAO,QAAQ,CAAC;IACpB,CAAC,CAAC;;AAUN;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,UAAU,gBAAgB,CAG9B,MAAwC;IACtC,OAAO,MAAM,CAAC,MAAM,CAAU,MAAM,CAAC,CAAC;AAC1C,CAAC"}
1
+ {"version":3,"file":"http-crawler.js","sourceRoot":"","sources":["../../src/internals/http-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,IAAI,MAAM,WAAW,CAAC;AAc7B,OAAO,EACH,YAAY,EACZ,oBAAoB,EACpB,aAAa,EACb,eAAe,EACf,YAAY,EACZ,yBAAyB,EACzB,YAAY,EACZ,eAAe,EACf,MAAM,EACN,YAAY,GACf,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EAAoB,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AACvE,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,OAAO,iBAAiB,MAAM,cAAc,CAAC;AAE7C,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,EAAE,MAAM,IAAI,CAAC;AAGpB,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAEhE,OAAO,EAAE,4BAA4B,EAAE,MAAM,YAAY,CAAC;AAE1D,IAAI,YAAsC,CAAC;AAE3C;;GAEG;AACH,MAAM,uBAAuB,GAAG,CAAC,WAAW,EAAE,UAAU,EAAE,uBAAuB,EAAE,iBAAiB,CAAC,CAAC;AACtG,MAAM,0BAA0B,GAAG,kBAAkB,CAAC;AACtD,MAAM,sCAAsC,GAA0B;IAClE,kBAAkB,EAAE,EAAE;IACtB,kBAAkB,EAAE;QAChB,6BAA6B,EAAE,CAAC;QAChC,gBAAgB,EAAE,GAAG;KACxB;IACD,mBAAmB,EAAE;QACjB,2BAA2B,EAAE,GAAG;KACnC;CACJ,CAAC;AA+LF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkEG;AACH,MAAM,OAAO,WAIX,SAAQ,YAAwD;IAmCxC;IAlCZ,kBAAkB,CAAsC;IACxD,mBAAmB,CAAuE;IAC1F,wBAAwB,CAAU;IAClC,uBAAuB,CAAS;IAChC,eAAe,CAAU;IACzB,uBAAuB,CAAU;IACjC,qBAAqB,CAAU;IAC/B,8BAA8B,CAAc;IAC5C,0BAA0B,CAAc;IAC/B,kBAAkB,CAAc;IAEzC,MAAM,CAAU,YAAY,GAAG;QACrC,GAAG,YAAY,CAAC,YAAY;QAE5B,qBAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QACpC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;QACxD,uBAAuB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC3C,qBAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,wBAAwB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QAE7C,8BAA8B,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;QACnE,0BAA0B,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;QAE/D,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK;QACrC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK;KACzC,CAAC;IAEF;;OAEG;IACH,YACI,UACmE,EAAS,EAC1D,SAAS,aAAa,CAAC,eAAe,EAAE;QAE1D,EAAE,CAAC,OAAO,EAAE,oBAAoB,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC;QAElF,MAAM,EACF,qBAAqB,GAAG,EAAE,EAC1B,eAAe,GAAG,IAAI,EACtB,mBAAmB,GAAG,EAAE,EACxB,uBAAuB,EACvB,qBAAqB,EACrB,wBAAwB,EACxB,kBAAkB,GAAG,EAAE,EACvB,mBAAmB,GAAG,EAAE,EACxB,8BAA8B,GAAG,EAAE,EACnC,0BAA0B,GAAG,EAAE;QAE/B,eAAe;QACf,qBAAqB,GAAG,sCAAsC,EAC9D,sBAAsB,EACtB,GAAG,mBAAmB,EACzB,GAAG,OAAO,CAAC;QAEZ,KAAK,CACD;YACI,GAAG,mBAAmB;YACtB,qBAAqB;YACrB,sBAAsB,EAClB,sBAAsB;gBACtB,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,oBAAoB,EAA+C,CAAC;SACvF,EACD,MAAM,CACT,CAAC;QA/BgB,WAAM,GAAN,MAAM,CAAkC;QAiC1D,uEAAuE;QACvE,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,wBAAwB,EAAE,CAAC;YACnD,MAAM,IAAI,KAAK,CAAC,iFAAiF,CAAC,CAAC;QACvG,CAAC;QAED,IAAI,CAAC,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,uBAAuB,EAAE,0BAA0B,CAAC,CAAC,CAAC;QAC5F,IAAI,mBAAmB,CAAC,MAAM;YAAE,IAAI,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC;QAEpF,IAAI,uBAAuB,IAAI,qBAAqB,EAAE,CAAC;YACnD,IAAI,CAAC,GAAG,CAAC,OAAO,CACZ,sGAAsG,CACzG,CAAC;QACN,CAAC;QAED,IAAI,CAAC,uBAAuB,GAAG,qBAAqB,GAAG,IAAI,CAAC;QAC5D,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,uBAAuB,GAAG,uBAAuB,CAAC;QACvD,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,CAAC;QACnD,IAAI,CAAC,8BAA8B,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,8BAA8B,CAAC,CAAC,CAAC;QACnF,IAAI,CAAC,0BAA0B,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,0BAA0B,CAAC,CAAC,CAAC;QAC3E,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,mBAAmB,GAAG;YACvB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,oBAAoB,CAAC,OAAO,EAAE,QAAS,CAAC;YACxE,GAAG,mBAAmB;SACzB,CAAC;QAEF,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,CAAC,wBAAwB,GAAG,wBAAwB,IAAI,IAAI,CAAC;QACrE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,wBAAwB,GAAG,KAAK,CAAC;QAC1C,CAAC;IACL,CAAC;IAES,oBAAoB;QAC1B,OAAO,eAAe,CAAC,MAAM,EAAmB;aAC3C,OAAO,CAAC;YACL,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC;SAC1C,CAAC;aACD,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;aACxD,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,6BAA6B,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC5E,CAAC;IAEO,KAAK,CAAC,eAAe,CACzB,eAAgC;QAEhC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAE7C,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YACzB,OAAO;gBACH,OAAO,EAAE,IAAI,KAAK,CAAC,OAAO,EAAE;oBACxB,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ;wBAC9B,IAAI,YAAY,KAAK,WAAW,EAAE,CAAC;4BAC/B,MAAM,IAAI,KAAK,CACX,+EAA+E,CAClF,CAAC;wBACN,CAAC;wBACD,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ,CAAC,CAAC;oBACvD,CAAC;iBACJ,CAAkC;gBACnC,IAAI,QAAQ;oBACR,MAAM,IAAI,KAAK,CAAC,sEAAsE,CAAC,CAAC;gBAC5F,CAAC;aACJ,CAAC;QACN,CAAC;QAED,MAAM,yBAAyB,GAAG,IAAI,CAAC,2BAA2B,CAAC,OAAO,CAAC,CAAC;QAE5E,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,UAAU,CAAC;QACxC,qEAAqE;QACrE,8CAA8C;QAC9C,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,kBAAkB,EAAE,eAAe,CAAC,CAAC;QACnE,SAAS,EAAE,CAAC;QAEZ,MAAM,0BAA0B,GAAG,IAAI,CAAC,2BAA2B,CAAC,OAAO,CAAC,CAAC;QAE7E,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,CAAC,eAAe,EAAE,yBAAyB,EAAE,0BAA0B,CAAC,CAAC;QAEhH,MAAM,QAAQ,GAAG,eAAe,CAAC,SAAS,EAAE,GAAG,CAAC;QAEhD,MAAM,YAAY,GAAG,MAAM,mBAAmB,CAC1C,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC,EAC/E,IAAI,CAAC,uBAAuB,EAC5B,2BAA2B,IAAI,CAAC,uBAAuB,GAAG,IAAI,WAAW,CAC5E,CAAC;QACF,SAAS,EAAE,CAAC;QAEZ,OAAO,CAAC,SAAS,GAAG,YAAY,EAAE,GAAG,CAAC;QACtC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,SAAS,CAAC;QAEvC,OAAO,EAAE,OAAO,EAAE,OAAwC,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC;IACzF,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAC7B,eAA2C;QAI3C,IAAI,eAAe,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YACzC,OAAO;gBACH,IAAI,WAAW;oBACX,MAAM,IAAI,KAAK,CAAC,yEAAyE,CAAC,CAAC;gBAC/F,CAAC;gBACD,IAAI,IAAI;oBACJ,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;gBACxF,CAAC;gBACD,IAAI,IAAI;oBACJ,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;gBACxF,CAAC;gBACD,IAAI,eAAe;oBACf,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAC;gBACjG,CAAC;gBACD,IAAI,gBAAgB;oBAChB,MAAM,IAAI,KAAK,CAAC,4EAA4E,CAAC,CAAC;gBAClG,CAAC;aACJ,CAAC;QACN,CAAC;QAED,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,mBAAmB,EAAE,eAAe,CAAC,CAAC;QACpE,SAAS,EAAE,CAAC;QAEZ,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,OAAO,EAAE,eAAe,CAAC,QAAQ,CAAC,CAAC;QAC5F,SAAS,EAAE,CAAC;QACZ,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAS,CAAC;QAClC,MAAM,WAAW,GAAG,MAAM,CAAC,WAAY,CAAC;QAExC,MAAM,eAAe,GAAG,KAAK,EAAE,QAAgB,EAAE,UAAmB,EAAE,EAAE;YACpE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;YAEhD,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACjC,MAAM,IAAI,KAAK,CAAC,aAAa,QAAQ,cAAc,CAAC,CAAC;YACzD,CAAC;QACL,CAAC,CAAC;QACF,MAAM,gBAAgB,GAAG,KAAK,EAAE,QAAiB,EAAE,SAAkB,EAAE,EAAE;YACrE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;YAEhD,IAAI,QAAQ,EAAE,CAAC;gBACX,MAAO,eAA+C,CAAC,eAAe,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;YAChG,CAAC;YAED,OAAO,CAAC,CAAC;QACb,CAAC,CAAC;QAEF,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,CAAC,sBAAsB,CAAC,eAAe,CAAC,OAAQ,EAAE,QAAQ,CAAC,MAAO,CAAC,CAAC;QAC5E,CAAC;QAED,IAAI,IAAI,CAAC,wBAAwB,EAAE,CAAC;YAChC,eAAe,CAAC,OAAQ,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC;QAC9D,CAAC;QAED,OAAO;YACH,IAAI,IAAI;gBACJ,IAAI,WAAW,CAAC,IAAI,KAAK,0BAA0B;oBAAE,OAAO,IAAI,CAAC;gBACjE,MAAM,UAAU,GAAG,MAAM,CAAC,IAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;gBAC/D,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAClC,CAAC;YACD,eAAe;YACf,gBAAgB;YAChB,WAAW;YACX,IAAI,EAAE,MAAM,CAAC,IAAI;SACpB,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,6BAA6B,CAAC,eAA4C;QACpF,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC;YAC3D,IAAI,KAAK;gBAAE,MAAM,IAAI,YAAY,CAAC,KAAK,CAAC,CAAC;QAC7C,CAAC;QACD,OAAO,EAAE,CAAC;IACd,CAAC;IAES,KAAK,CAAC,gBAAgB,CAAC,eAA4C;QACzE,IAAI,uBAAuB,CAAC,QAAQ,CAAC,eAAe,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;YACrE,MAAM,CAAC,GAAG,MAAM,eAAe,CAAC,gBAAgB,EAAE,CAAC;YAEnD,MAAM,cAAc,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAExF,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,OAAO,oBAAoB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3D,CAAC;QACL,CAAC;QAED,MAAM,kBAAkB;QACpB,wCAAwC;QACxC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,oBAAoB,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC;YACtD,CAAC,CAAC,wCAAwC;gBACxC,IAAI,CAAC,WAAY,CAAC,oBAAoB,CAAC;YACzC,CAAC,CAAC,oBAAoB,CAAC;QAE/B,IAAI,kBAAkB,CAAC,QAAQ,CAAC,eAAe,CAAC,QAAQ,CAAC,MAAO,CAAC,EAAE,CAAC;YAChE,OAAO,0BAA0B,eAAe,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;QACvE,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;OAGG;IACO,aAAa,CACnB,EAAE,OAAO,EAAE,OAAO,EAAmB,EACrC,cAAsB,EACtB,eAAuB;QAEvB,MAAM,aAAa,GAAG,OAAO,EAAE,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QAElE,MAAM,aAAa,GAAG,CAAC,aAAa,EAAE,cAAc,EAAE,eAAe,CAAC,CAAC;QAEvE,OAAO,YAAY,CAAC,OAAO,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;IACpD,CAAC;IAED;;;;OAIG;IACO,KAAK,CAAC,gBAAgB,CAAC,EAC7B,OAAO,EACP,OAAO,EACP,QAAQ,EACR,YAAY,GACS;QACrB,IAAI,CAAC,YAAY,EAAE,CAAC;YAChB,aAAa;YACb,CAAC,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC;QACtD,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;QAEjE,IAAI,CAAC;YACD,OAAO,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC;QACrE,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,YAAY,YAAY,EAAE,CAAC;gBAC5B,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;gBACpC,OAAO,IAAI,QAAQ,EAAE,CAAC,CAAC,iEAAiE;YAC5F,CAAC;YAED,IAAI,IAAI,CAAC,YAAY,CAAC,CAAU,CAAC,EAAE,CAAC;gBAChC,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAU,CAAW,CAAC,CAAC;YAC5E,CAAC;iBAAM,CAAC;gBACJ,MAAM,CAAC,CAAC;YACZ,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,cAAc,CAAC,OAAuB,EAAE,QAAkB;QACtE,MAAM,EAAE,MAAM,EAAE,GAAG,QAAQ,CAAC;QAC5B,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,4BAA4B,CAAC,QAAQ,CAAC,CAAC;QACjE,MAAM,EAAE,QAAQ,EAAE,iBAAiB,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;QACnG,MAAM,WAAW,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QAEvC,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;YACjC,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;QAC1C,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,CAAC,0BAA0B,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACjE,MAAM,YAAY,GAAG,IAAI,CAAC,8BAA8B,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAErE,IAAI,CAAC,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,YAAY,EAAE,CAAC;YACnD,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,IAAI,EAAE,CAAC,CAAC,oGAAoG;YAEjJ,2DAA2D;YAC3D,gDAAgD;YAChD,IAAI,IAAI,KAAK,0BAA0B,EAAE,CAAC;gBACtC,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACvC,IAAI,EAAE,OAAO,EAAE,GAAG,aAAa,CAAC;gBAChC,IAAI,CAAC,OAAO;oBAAE,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,cAAc,EAAE,EAAE,EAAE,CAAC,CAAC;gBACtF,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,MAAM,OAAO,EAAE,CAAC,CAAC;YAC9C,CAAC;YAED,IAAI,YAAY,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,uCAAuC,CAAC,CAAC;YACtE,CAAC;YAED,8EAA8E;YAC9E,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,6BAA6B,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAChF,CAAC;aAAM,IAAI,uBAAuB,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAChD,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;QAClE,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;YACjD,OAAO;gBACH,IAAI;gBACJ,QAAQ;gBACR,WAAW;aACd,CAAC;QACN,CAAC;IACL,CAAC;IAED;;OAEG;IACO,kBAAkB,CAAC,OAAuB,EAAE,OAAiB,EAAE,QAAiB;QACtF,MAAM,cAAc,GAAG;YACnB,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,QAAQ;YACR,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,uBAAuB,EAAE;YAClD,SAAS,EAAE,IAAI,CAAC,wBAAwB,CAAC,CAAC,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,SAAS;YACzE,YAAY,EAAE,OAAO;YACrB,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,KAAK,EAAE;gBACH,kBAAkB,EAAE,CAAC,IAAI,CAAC,eAAe;aAC5C;YACD,IAAI,EAAE,SAA+B;SACxC,CAAC;QAEF,yHAAyH;QACzH,OAAO,CAAC,cAAc,CAAC,cAAc,CAAC,OAAQ,EAAE,QAAQ,CAAC,CAAC;QAE1D,4CAA4C;QAC5C,IAAI,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,CAAC;YACtC,cAAc,CAAC,KAAK,GAAG;gBACnB,GAAG,cAAc,CAAC,KAAK;gBACvB,kBAAkB,EAAE,KAAK;aAC5B,CAAC;QACN,CAAC;QAED,IAAI,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,cAAc,CAAC,IAAI,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;QAEvF,OAAO,cAAc,CAAC;IAC1B,CAAC;IAES,eAAe,CACrB,OAAuB,EACvB,QAAkB,EAClB,QAAwB;QAKxB,IAAI,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAC7B,QAAQ,GAAG,IAAI,CAAC,qBAAuC,CAAC;QAC5D,CAAC;aAAM,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,uBAAuB,EAAE,CAAC;YACnD,QAAQ,GAAG,IAAI,CAAC,uBAAyC,CAAC;QAC9D,CAAC;QAED,sDAAsD;QACtD,MAAM,IAAI,GAAG,MAAM,CAAC;QACpB,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAEnD,2DAA2D;QAC3D,+CAA+C;QAC/C,IAAI,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;QAE/D,+DAA+D;QAC/D,IAAI,KAAK,CAAC,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC;YACjC,MAAM,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YACxG,MAAM,aAAa,GAAG,QAAQ,CAAC,IAAI;gBAC/B,CAAC,CAAC,QAAQ,CAAC,KAAK,CACV,QAAQ,CAAC,IAAI,CACT,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAW,CAAC;qBACjC,IAAI,CAAC,YAAY,CAAC;qBAClB,IAAI,CAAC,YAAY,CAAC,CAC1B,CACJ;gBACH,CAAC,CAAC,IAAI,CAAC;YAEX,OAAO;gBACH,QAAQ,EAAE,IAAI,eAAe,CAAC,aAAoB,EAAE,QAAQ,CAAC;gBAC7D,QAAQ,EAAE,IAAI;aACjB,CAAC;QACN,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,YAAY,OAAO,CAAC,GAAG,8CAA8C,QAAQ,EAAE,CAAC,CAAC;IACrG,CAAC;IAED;;OAEG;IACO,yBAAyB,CAAC,mBAA4D;QAC5F,KAAK,MAAM,QAAQ,IAAI,mBAAmB,EAAE,CAAC;YACzC,IAAI,QAAQ,KAAK,KAAK,EAAE,CAAC;gBACrB,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBACtC,SAAS;YACb,CAAC;YAED,IAAI,CAAC;gBACD,MAAM,UAAU,GAAG,iBAAiB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBACrD,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;YACjD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,IAAI,KAAK,CAAC,2BAA2B,QAAQ,sCAAsC,CAAC,CAAC;YAC/F,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACO,qBAAqB,CAAC,OAAiB;QAC7C,OAAO,EAAE,OAAO,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,uBAAuB,GAAG,IAAI,WAAW,CAAC,CAAC;IAC/F,CAAC;IAEO,oBAAoB,CAAC,OAAuB,EAAE,QAAkB;QACpE,MAAM,EAAE,MAAM,EAAE,GAAG,QAAQ,CAAC;QAC5B,MAAM,EAAE,IAAI,EAAE,GAAG,4BAA4B,CAAC,QAAQ,CAAC,CAAC;QAExD,sEAAsE;QACtE,MAAM,kBAAkB,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1F,wDAAwD;QACxD,MAAM,sBAAsB,GAAG,MAAM,IAAI,GAAG,IAAI,kBAAkB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAEpF,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;YACvG,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;YACvB,MAAM,IAAI,KAAK,CACX,YAAY,OAAO,CAAC,GAAG,wBAAwB,IAAI,IAAI;gBACnD,YAAY,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,kCAAkC,CACnG,CAAC;QACN,CAAC;IACL,CAAC;IAED;;OAEG;IACK,iBAAiB,GAAG,KAAK,EAAE,OAAwB,EAAE,OAAiB,EAAE,YAAqB,EAAE,EAAE;QACrG,MAAM,IAAI,GAAG,yBAAyB,CAAC;YACnC,GAAI,OAAe;YACnB,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,YAAY,EAAE,MAAM;SACvB,CAAC,CAAC;QAEH,IAAI,YAAY,EAAE,CAAC;YACf,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC/B,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC/B,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QAC9C,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CACzC,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE;YAClB,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAS,CAAC,CAAC,CAAC,SAAS;YAChE,OAAO,EAAE,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;YAClC,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,kEAAkE;YAClE,MAAM,EAAE,MAAM;SACF,CAAC,EACjB;YACI,OAAO;YACP,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,UAAU,EAAE,CAAC,gBAAgB,EAAE,cAAc,EAAE,EAAE;gBAC7C,IAAI,IAAI,CAAC,wBAAwB,EAAE,CAAC;oBAChC,OAAQ,CAAC,sBAAsB,CAAC,gBAAgB,CAAC,CAAC;oBAElD,MAAM,sBAAsB,GAAG,OAAQ,CAAC,eAAe,CAAC,cAAc,CAAC,GAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;oBACxF,IAAI,sBAAsB,KAAK,EAAE,EAAE,CAAC;wBAChC,cAAc,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,sBAAsB,CAAC,CAAC;oBACjE,CAAC;gBACL,CAAC;YACL,CAAC;SACJ,CACJ,CAAC;QAEF,OAAO,QAAQ,CAAC;IACpB,CAAC,CAAC;;AAUN;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,UAAU,gBAAgB,CAG9B,MAAwC;IACtC,OAAO,MAAM,CAAC,MAAM,CAAU,MAAM,CAAC,CAAC;AAC1C,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/http",
3
- "version": "4.0.0-beta.17",
3
+ "version": "4.0.0-beta.19",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
6
  "node": ">=22.0.0"
@@ -49,9 +49,9 @@
49
49
  "dependencies": {
50
50
  "@apify/timeout": "^0.3.2",
51
51
  "@apify/utilities": "^2.15.5",
52
- "@crawlee/basic": "4.0.0-beta.17",
53
- "@crawlee/types": "4.0.0-beta.17",
54
- "@crawlee/utils": "4.0.0-beta.17",
52
+ "@crawlee/basic": "4.0.0-beta.19",
53
+ "@crawlee/types": "4.0.0-beta.19",
54
+ "@crawlee/utils": "4.0.0-beta.19",
55
55
  "@types/content-type": "^1.1.8",
56
56
  "cheerio": "^1.0.0",
57
57
  "content-type": "^1.0.5",
@@ -69,5 +69,5 @@
69
69
  }
70
70
  }
71
71
  },
72
- "gitHead": "755dcbfced2107997ea4d4e9fdf1839e141d3f00"
72
+ "gitHead": "690af03140bea34c9a8871d370607cd52339caa0"
73
73
  }