@crawlee/http 4.0.0-beta.66 → 4.0.0-beta.67

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (10) hide show
  1. package/README.md +10 -10
  2. package/internals/http-crawler.d.ts +1 -1
  3. package/internals/http-crawler.d.ts.map +1 -1
  4. package/internals/http-crawler.js +10 -1
  5. package/internals/http-crawler.js.map +1 -1
  6. package/internals/utils.d.ts +6 -0
  7. package/internals/utils.d.ts.map +1 -1
  8. package/internals/utils.js +11 -0
  9. package/internals/utils.js.map +1 -1
  10. package/package.json +7 -7
package/README.md CHANGED
@@ -1,23 +1,23 @@
1
1
  <h1 align="center">
2
2
  <a href="https://crawlee.dev">
3
3
  <picture>
4
- <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true">
5
- <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500">
4
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true" />
5
+ <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500" />
6
6
  </picture>
7
7
  </a>
8
- <br>
8
+ <br />
9
9
  <small>A web scraping and browser automation library</small>
10
10
  </h1>
11
11
 
12
- <p align=center>
13
- <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
12
+ <p align="center">
13
+ <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" width="250" height="55"/></a>
14
14
  </p>
15
15
 
16
- <p align=center>
17
- <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" style="max-width: 100%;"></a>
18
- <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" style="max-width: 100%;"></a>
19
- <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" style="max-width: 100%;"></a>
20
- <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" style="max-width: 100%;"></a>
16
+ <p align="center">
17
+ <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" /></a>
18
+ <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" /></a>
19
+ <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" /></a>
20
+ <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" /></a>
21
21
  </p>
22
22
 
23
23
  Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
package/internals/http-crawler.d.ts CHANGED
@@ -303,7 +303,7 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
303
303
  // @ts-ignore optional peer dependency or compatibility with es2022
304
304
  configuration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
305
305
  // @ts-ignore optional peer dependency or compatibility with es2022
306
- storageClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
306
+ storageBackend: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
307
307
  // @ts-ignore optional peer dependency or compatibility with es2022
308
308
  eventManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
309
309
  // @ts-ignore optional peer dependency or compatibility with es2022
package/internals/http-crawler.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"http-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/http-crawler.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAER,mBAAmB,EAEnB,eAAe,EACf,YAAY,EACZ,sBAAsB,EACtB,OAAO,IAAI,cAAc,EACzB,cAAc,EACd,sBAAsB,EACtB,YAAY,EACf,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACH,YAAY,EACZ,eAAe,EAKlB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,KAAK,aAAa,EAA0B,MAAM,eAAe,CAAC;AAE3E,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AACtE,OAAO,EAAE,KAAK,WAAW,EAAuB,MAAM,gBAAgB,CAAC;AAEvE,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAI9D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAsB3C,MAAM,MAAM,gBAAgB,CACxB,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,IAChC,YAAY,CAAC,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE1D,MAAM,WAAW,kBAAkB,CAC/B,OAAO,SAAS,2BAA2B,GAAG,2BAA2B,EACzE,gBAAgB,GAAG,UAAU,CAAC,KAAK,CAAC,EACpC,eAAe,SAAS,OAAO,GAAG,OAAO,GAAG,gBAAgB,CAC9D,SAAQ,mBAAmB,CAAC,OAAO,EAAE,gBAAgB,EAAE,eAAe,CAAC;IACrE;;OAEG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;OAEG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;;;;;;;;;;;;;;OAeG;IACH,kBAAkB,CAAC,EAAE,gBAAgB,CAAC,eAAe,CAAC,EAAE,CAAC;IAEzD;;;;;;;;;;;;;;;;OAgBG;IACH,mBAAmB,CAAC,EAAE,CAAC,CACnB,eAAe,EAAE,2BAA2B,KAC3C,SAAS,CAAC,IAAI,GAAG,OAAO,CAAC,2BAA2B,CAAC,CAAC,CAAC,EAAE,CAAC;IAE/D;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE/B;;;;;;;;;;OAUG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAEjC;;;;;;;;OAQG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;;;OAKG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAC;CACjC;AAED;;GAEG;AACH,MAAM,MAAM,gBAAgB,CAAC,OAAO,IAAI,CAAC,eAAe,EAAE,OAAO,KAAK,SAAS,CAAC,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;AAEzG,MAAM,MAAM,QAAQ,CAChB,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,IAChC,gBAAgB,CAAC,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE9D,UAAU,2BAA2B,CACjC,QAAQ,SAAS,UAAU,GAAG,GAAG,CACnC,SAAQ,eAAe,CAAC,QAAQ,CAAC;IAC/B;;OAEG;IACH,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,CAAC;IAEjD;;OAEG;IACH,QAAQ,EAAE,QAAQ,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,2BAA2B,CACxC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,CAClC,SAAQ,2BAA2B,CAAC,QAAQ,CAAC;IAC3C;;;;;OAKG;IACH,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;IAEtB;;OAEG;IACH,IAAI,EAAE,QAAQ,CAAC;IAEf;;OAEG;IACH,WAAW,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,cAAc,CAAA;KAAE,CAAC;IAExD;;;;;;;;;;;OAWG;IACH,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErE;;;;;;;;;;;OAWG;IACH,gBAAgB,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;CACjF;AAED,MAAM,WAAW,mBAAmB,CAChC,QAAQ,SAAS,UAAU,GAAG,GAAG,EACjC,QAAQ,SAAS,SAAS,GAAG,GAAG,CAClC,SAAQ,2BAA2B,CAAC,QAAQ,EAAE,QAAQ,CAAC;CAAG;AAE5D,MAAM,MAAM,kBAAkB,CAC1B,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,IAChC,cAAc,CAAC,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoEG;AACH,qBAAa,WAAW,CACpB,OAAO,SAAS,2BAA2B,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,2BAA2B,EACnF,gBAAgB,GAAG,UAAU,CAAC,KAAK,CAAC,EACpC,eAAe,SAAS,OAAO,GAAG,OAAO,GAAG,gBAAgB,CAC9D,SAAQ,YAAY,CAAC,OAAO,EAAE,gBAAgB,EAAE,eAAe,CAAC;IAC9D,SAAS,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,eAAe,CAAC,EAAE,CAAC;IAClE,SAAS,CAAC,mBAAmB,EAAE,CAAC,CAC5B,eAAe,EAAE,2BAA2B,KAC3C,SAAS,CAAC,IAAI,GAAG,OAAO,CAAC,2BAA2B,CAAC,CAAC,CAAC,EAAE,CAAC;IAC/D,SAAS,CAAC,mBAAmB,EAAE,OAAO,CAAC;IACvC,SAAS,CAAC,uBAAuB,EAAE,MAAM,CAAC;IAC1C,SAAS,CAAC,eAAe,EAAE,OAAO,CAAC;IACnC,SAAS,CAAC,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAC3C,SAAS,CAAC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IACzC,SAAS,CAAC,QAAQ,CAAC,kBAAkB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAEnD,iBAA0B,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAYpC;IAEF;;OAEG;gBAEC,OAAO,GAAE,kBAAkB,CAAC,OAAO,EAAE,gBAAgB,EAAE,eAAe,CAAC,GACnE,sBAAsB,CAAC,2BAA2B,EAAE,OAAO,CAAa;cAkD7D,oBAAoB,IAAI,eAAe,CAAC,eAAe,EAAE,2BAA2B,CAAC;YA4B1F,kBAAkB;YA2BlB,eAAe;YAqBf,mBAAmB;YAyFnB,6BAA6B;cAQ3B,gBAAgB,CAAC,eAAe,EAAE,2BAA2B,GAAG,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC;IAkBvG;;;;OAIG;cACa,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,sBAAsB,GAAG,OAAO,CAAC,QAAQ,CAAC;IAmB3G;;OAEG;cACa,cAAc,CAAC,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,QAAQ;;;;;;;;;;;;;;;IAwC1E;;OAEG;IACH,SAAS,CAAC,kBAAkB,CAAC,OAAO,EAAE,cAAc,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,EAAE,MAAM;;;;;;;;;;cAW/D,MAAM,GAAG,SAAS;;IAqB7C,SAAS,CAAC,eAAe,CACrB,OAAO,EAAE,cAAc,EACvB,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,cAAc,GACzB;QACC,QAAQ,EAAE,cAAc,CAAC;QACzB,QAAQ,EAAE,QAAQ,CAAC;KACtB;IAwCD;;OAEG;IACH,SAAS,CAAC,yBAAyB,CAAC,mBAAmB,EAAE,CAAC,MAAM,GAAG,WAAW,GAAG,YAAY,CAAC,EAAE;IAgBhG;;OAEG;IACH,SAAS,CAAC,qBAAqB,CAAC,OAAO,EAAE,QAAQ;IAKjD,OAAO,CAAC,oBAAoB;IAe5B;;OAEG;IACH,OAAO,CAAC,iBAAiB,CA2BvB;CACL;AAED,UAAU,sBAAsB;IAC5B,OAAO,EAAE,cAAc,CAAC;IACxB,OAAO,EAAE,QAAQ,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,gBAAgB,CAC5B,OAAO,SAAS,mBAAmB,GAAG,mBAAmB,EACzD,QAAQ,SAAS,UAAU,GAAG,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAC1E,MAAM,CAAC,EAAE,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,mDAEzC"}
1
+ {"version":3,"file":"http-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/http-crawler.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAER,mBAAmB,EAEnB,eAAe,EACf,YAAY,EACZ,sBAAsB,EACtB,OAAO,IAAI,cAAc,EACzB,cAAc,EACd,sBAAsB,EACtB,YAAY,EACf,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACH,YAAY,EACZ,eAAe,EAKlB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,KAAK,aAAa,EAA0B,MAAM,eAAe,CAAC;AAE3E,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AACtE,OAAO,EAAE,KAAK,WAAW,EAAuB,MAAM,gBAAgB,CAAC;AAEvE,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAI9D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAsB3C,MAAM,MAAM,gBAAgB,CACxB,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,IAChC,YAAY,CAAC,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE1D,MAAM,WAAW,kBAAkB,CAC/B,OAAO,SAAS,2BAA2B,GAAG,2BAA2B,EACzE,gBAAgB,GAAG,UAAU,CAAC,KAAK,CAAC,EACpC,eAAe,SAAS,OAAO,GAAG,OAAO,GAAG,gBAAgB,CAC9D,SAAQ,mBAAmB,CAAC,OAAO,EAAE,gBAAgB,EAAE,eAAe,CAAC;IACrE;;OAEG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;OAEG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;;;;;;;;;;;;;;OAeG;IACH,kBAAkB,CAAC,EAAE,gBAAgB,CAAC,eAAe,CAAC,EAAE,CAAC;IAEzD;;;;;;;;;;;;;;;;OAgBG;IACH,mBAAmB,CAAC,EAAE,CAAC,CACnB,eAAe,EAAE,2BAA2B,KAC3C,SAAS,CAAC,IAAI,GAAG,OAAO,CAAC,2BAA2B,CAAC,CAAC,CAAC,EAAE,CAAC;IAE/D;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE/B;;;;;;;;;;OAUG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAEjC;;;;;;;;OAQG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;;;OAKG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAC;CACjC;AAED;;GAEG;AACH,MAAM,MAAM,gBAAgB,CAAC,OAAO,IAAI,CAAC,eAAe,EAAE,OAAO,KAAK,SAAS,CAAC,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;AAEzG,MAAM,MAAM,QAAQ,CAChB,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,IAChC,gBAAgB,CAAC,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE9D,UAAU,2BAA2B,CACjC,QAAQ,SAAS,UAAU,GAAG,GAAG,CACnC,SAAQ,eAAe,CAAC,QAAQ,CAAC;IAC/B;;OAEG;IACH,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,CAAC;IAEjD;;OAEG;IACH,QAAQ,EAAE,QAAQ,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,2BAA2B,CACxC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,CAClC,SAAQ,2BAA2B,CAAC,QAAQ,CAAC;IAC3C;;;;;OAKG;IACH,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;IAEtB;;OAEG;IACH,IAAI,EAAE,QAAQ,CAAC;IAEf;;OAEG;IACH,WAAW,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,cAAc,CAAA;KAAE,CAAC;IAExD;;;;;;;;;;;OAWG;IACH,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErE;;;;;;;;;;;OAWG;IACH,gBAAgB,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;CACjF;AAED,MAAM,WAAW,mBAAmB,CAChC,QAAQ,SAAS,UAAU,GAAG,GAAG,EACjC,QAAQ,SAAS,SAAS,GAAG,GAAG,CAClC,SAAQ,2BAA2B,CAAC,QAAQ,EAAE,QAAQ,CAAC;CAAG;AAE5D,MAAM,MAAM,kBAAkB,CAC1B,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,IAChC,cAAc,CAAC,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoEG;AACH,qBAAa,WAAW,CACpB,OAAO,SAAS,2BAA2B,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,2BAA2B,EACnF,gBAAgB,GAAG,UAAU,CAAC,KAAK,CAAC,EACpC,eAAe,SAAS,OAAO,GAAG,OAAO,GAAG,gBAAgB,CAC9D,SAAQ,YAAY,CAAC,OAAO,EAAE,gBAAgB,EAAE,eAAe,CAAC;IAC9D,SAAS,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,eAAe,CAAC,EAAE,CAAC;IAClE,SAAS,CAAC,mBAAmB,EAAE,CAAC,CAC5B,eAAe,EAAE,2BAA2B,KAC3C,SAAS,CAAC,IAAI,GAAG,OAAO,CAAC,2BAA2B,CAAC,CAAC,CAAC,EAAE,CAAC;IAC/D,SAAS,CAAC,mBAAmB,EAAE,OAAO,CAAC;IACvC,SAAS,CAAC,uBAAuB,EAAE,MAAM,CAAC;IAC1C,SAAS,CAAC,eAAe,EAAE,OAAO,CAAC;IACnC,SAAS,CAAC,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAC3C,SAAS,CAAC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IACzC,SAAS,CAAC,QAAQ,CAAC,kBAAkB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAEnD,iBAA0B,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAYpC;IAEF;;OAEG;gBAEC,OAAO,GAAE,kBAAkB,CAAC,OAAO,EAAE,gBAAgB,EAAE,eAAe,CAAC,GACnE,sBAAsB,CAAC,2BAA2B,EAAE,OAAO,CAAa;cAkD7D,oBAAoB,IAAI,eAAe,CAAC,eAAe,EAAE,2BAA2B,CAAC;YA4B1F,kBAAkB;YA2BlB,eAAe;YAqBf,mBAAmB;YAyFnB,6BAA6B;cAQ3B,gBAAgB,CAAC,eAAe,EAAE,2BAA2B,GAAG,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC;IAkBvG;;;;OAIG;cACa,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,sBAAsB,GAAG,OAAO,CAAC,QAAQ,CAAC;IAmB3G;;OAEG;cACa,cAAc,CAAC,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,QAAQ;;;;sBAoCD,cAAc;;;;;;;;;;;IAavF;;OAEG;IACH,SAAS,CAAC,kBAAkB,CAAC,OAAO,EAAE,cAAc,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,EAAE,MAAM;;;;;;;;;;cAW/D,MAAM,GAAG,SAAS;;IAqB7C,SAAS,CAAC,eAAe,CACrB,OAAO,EAAE,cAAc,EACvB,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,cAAc,GACzB;QACC,QAAQ,EAAE,cAAc,CAAC;QACzB,QAAQ,EAAE,QAAQ,CAAC;KACtB;IAwCD;;OAEG;IACH,SAAS,CAAC,yBAAyB,CAAC,mBAAmB,EAAE,CAAC,MAAM,GAAG,WAAW,GAAG,YAAY,CAAC,EAAE;IAgBhG;;OAEG;IACH,SAAS,CAAC,qBAAqB,CAAC,OAAO,EAAE,QAAQ;IAKjD,OAAO,CAAC,oBAAoB;IAe5B;;OAEG;IACH,OAAO,CAAC,iBAAiB,CA2BvB;CACL;AAED,UAAU,sBAAsB;IAC5B,OAAO,EAAE,cAAc,CAAC;IACxB,OAAO,EAAE,QAAQ,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,gBAAgB,CAC5B,OAAO,SAAS,mBAAmB,GAAG,mBAAmB,EACzD,QAAQ,SAAS,UAAU,GAAG,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAC1E,MAAM,CAAC,EAAE,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,mDAEzC"}
package/internals/http-crawler.js CHANGED
@@ -9,7 +9,7 @@ import contentTypeParser from 'content-type';
9
9
  import iconv from 'iconv-lite';
10
10
  import ow from 'ow';
11
11
  import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
12
- import { parseContentTypeFromResponse, processHttpRequestOptions } from './utils.js';
12
+ import { extractCharsetFromHtmlBytes, parseContentTypeFromResponse, processHttpRequestOptions } from './utils.js';
13
13
  /**
14
14
  * Default mime types, which HttpScraper supports.
15
15
  */
@@ -337,6 +337,15 @@ export class HttpCrawler extends BasicCrawler {
337
337
  throw new Error(`${status} - Internal Server Error: ${body.slice(0, 100)}`);
338
338
  }
339
339
  else if (HTML_AND_XML_MIME_TYPES.includes(type)) {
340
+ if (!charset && !this.forceResponseEncoding) {
341
+ const rawBytes = Buffer.from(await response.arrayBuffer());
342
+ const metaCharset = extractCharsetFromHtmlBytes(rawBytes);
343
+ const charsetToUse = metaCharset ?? this.suggestResponseEncoding ?? 'utf-8';
344
+ const body = iconv.encodingExists(charsetToUse)
345
+ ? iconv.decode(rawBytes, charsetToUse)
346
+ : rawBytes.toString('utf8');
347
+ return { response, contentType: { type, encoding: 'utf-8' }, body };
348
+ }
340
349
  return { response, contentType, body: await reencodedResponse.text() };
341
350
  }
342
351
  else {
package/internals/http-crawler.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"http-crawler.js","sourceRoot":"","sources":["../../src/internals/http-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,IAAI,MAAM,WAAW,CAAC;AAc7B,OAAO,EACH,YAAY,EACZ,eAAe,EACf,sBAAsB,EACtB,YAAY,EACZ,MAAM,EACN,YAAY,GACf,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAsB,sBAAsB,EAAE,MAAM,eAAe,CAAC;AAC3E,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAEvD,OAAO,EAAoB,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AACvE,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,OAAO,iBAAiB,MAAM,cAAc,CAAC;AAC7C,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,EAAE,MAAM,IAAI,CAAC;AAGpB,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAEhE,OAAO,EAAE,4BAA4B,EAAE,yBAAyB,EAAE,MAAM,YAAY,CAAC;AAErF;;GAEG;AACH,MAAM,uBAAuB,GAAG,CAAC,WAAW,EAAE,UAAU,EAAE,uBAAuB,EAAE,iBAAiB,CAAC,CAAC;AACtG,MAAM,0BAA0B,GAAG,kBAAkB,CAAC;AACtD,MAAM,sCAAsC,GAA0B;IAClE,kBAAkB,EAAE,EAAE;IACtB,kBAAkB,EAAE;QAChB,6BAA6B,EAAE,CAAC;QAChC,gBAAgB,EAAE,GAAG;KACxB;IACD,mBAAmB,EAAE;QACjB,2BAA2B,EAAE,GAAG;KACnC;CACJ,CAAC;AA6LF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoEG;AACH,MAAM,OAAO,WAIX,SAAQ,YAAwD;IACpD,kBAAkB,CAAsC;IACxD,mBAAmB,CAEkC;IACrD,mBAAmB,CAAU;IAC7B,uBAAuB,CAAS;IAChC,eAAe,CAAU;IACzB,uBAAuB,CAAU;IACjC,qBAAqB,CAAU;IACtB,kBAAkB,CAAc;IAEzC,MAAM,CAAU,YAAY,GAAG;QACrC,GAAG,YAAY,CAAC,YAAY;QAE5B,qBAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QACpC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;QACxD,uBAAuB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC3C,qBAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QAExC,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK;QACrC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK;KACzC,CAAC;IAEF;;OAEG;IACH,YACI,UACmE,EAAS;QAE5E,EAAE,CAAC,OAAO,EAAE,oBAAoB,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC;QAElF,MAAM,EACF,qBAAqB,GAAG,EAAE,EAC1B,eAAe,GAAG,IAAI,EACtB,mBAAmB,GAAG,EAAE,EACxB,uBAAuB,EACvB,qBAAqB,EACrB,mBAAmB,GAAG,IAAI,EAC1B,kBAAkB,GAAG,EAAE,EACvB,mBAAmB,GAAG,EAAE;QAExB,eAAe;QACf,qBAAqB,GAAG,sCAAsC,EAC9D,sBAAsB,EACtB,GAAG,mBAAmB,EACzB,GAAG,OAAO,CAAC;QAEZ,KAAK,CAAC;YACF,GAAG,mBAAmB;YACtB,qBAAqB;YACrB,sBAAsB,EAClB,sBAAsB;gBACtB,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,oBAAoB,EAA+C,CAAC;SACvF,CAAC,CAAC;QAEH,IAAI,CAAC,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,uBAAuB,EAAE,0BAA0B,CAAC,CAAC,CAAC;QAC5F,IAAI,mBAAmB,CAAC,MAAM;YAAE,IAAI,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC;QAEpF,IAAI,uBAAuB,IAAI,qBAAqB,EAAE,CAAC;YACnD,IAAI,CAAC,GAAG,CAAC,OAAO,CACZ,sGAAsG,CACzG,CAAC;QACN,CAAC;QAED,IAAI,CAAC,uBAAuB,GAAG,qBAAqB,GAAG,IAAI,CAAC;QAC5D,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,uBAAuB,GAAG,uBAAuB,CAAC;QACvD,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,CAAC;QACnD,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,mBAAmB,GAAG;YACvB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,oBAAoB,CAAC,OAAO,EAAE,QAAS,CAAC;YACxE,GAAG,mBAAmB;SACzB,CAAC;QAEF,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;IACnD,CAAC;IAEkB,oBAAoB;QACnC,8FAA8F;QAC9F,kGAAkG;QAClG,MAAM,SAAS,GAAG,CACd,MAA2C,EAChB,EAAE,CAAC,CAAC;YAC/B,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAQ;SAChG,CAAC,CAAC;QAEH,IAAI,QAAQ,GAAG,eAAe,CAAC,MAAM,EAAmB,CAAC,OAAO,CAAC;YAC7D,MAAM,EAAE,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC;SAC7C,CAAC,CAAC;QAEH,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACzC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;QACjD,CAAC;QAED,IAAI,sBAAsB,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAE1F,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC1C,sBAAsB,GAAG,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;QAC7E,CAAC;QAED,OAAO,sBAAsB;aACxB,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;aACxD,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,6BAA6B,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC5E,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,eAAgC;QAC7D,MAAM,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAEpC,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YACzB,OAAO;gBACH,OAAO,EAAE,IAAI,KAAK,CAAC,OAAO,EAAE;oBACxB,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ;wBAC9B,IAAI,YAAY,KAAK,WAAW,EAAE,CAAC;4BAC/B,MAAM,IAAI,sBAAsB,CAC5B,+EAA+E,CAClF,CAAC;wBACN,CAAC;wBACD,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ,CAAC,CAAC;oBACvD,CAAC;iBACJ,CAAkC;gBACnC,IAAI,QAAQ;oBACR,MAAM,IAAI,sBAAsB,CAC5B,sEAAsE,CACzE,CAAC;gBACN,CAAC;aACoC,CAAC;QAC9C,CAAC;QAED,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,UAAU,CAAC;QACxC,OAAO,EAAE,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,eAAe,CACzB,eAAgC;QAEhC,SAAS,EAAE,CAAC;QAEZ,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAC7C,MAAM,QAAQ,GAAG,eAAe,CAAC,SAAS,EAAE,GAAG,CAAC;QAEhD,MAAM,YAAY,GAAG,MAAM,mBAAmB,CAC1C,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,EACjE,IAAI,CAAC,uBAAuB,EAC5B,2BAA2B,IAAI,CAAC,uBAAuB,GAAG,IAAI,WAAW,CAC5E,CAAC;QACF,SAAS,EAAE,CAAC;QAEZ,OAAO,CAAC,SAAS,GAAG,YAAY,EAAE,GAAG,CAAC;QACtC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,SAAS,CAAC;QAEvC,OAAO,EAAE,OAAO,EAAE,OAAwC,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC;IACzF,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAC7B,eAA4C;QAI5C,IAAI,eAAe,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YACzC,OAAO;gBACH,IAAI,WAAW;oBACX,MAAM,IAAI,sBAAsB,CAC5B,yEAAyE,CAC5E,CAAC;gBACN,CAAC;gBACD,IAAI,IAAI;oBACJ,MAAM,IAAI,sBAAsB,CAC5B,kEAAkE,CACrE,CAAC;gBACN,CAAC;gBACD,IAAI,IAAI;oBACJ,MAAM,IAAI,sBAAsB,CAC5B,kEAAkE,CACrE,CAAC;gBACN,CAAC;gBACD,IAAI,eAAe;oBACf,MAAM,IAAI,sBAAsB,CAC5B,2EAA2E,CAC9E,CAAC;gBACN,CAAC;gBACD,IAAI,gBAAgB;oBAChB,MAAM,IAAI,sBAAsB,CAC5B,4EAA4E,CAC/E,CAAC;gBACN,CAAC;aACJ,CAAC;QACN,CAAC;QAED,SAAS,EAAE,CAAC;QAEZ,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,OAAO,EAAE,eAAe,CAAC,QAAQ,CAAC,CAAC;QAC5F,SAAS,EAAE,CAAC;QACZ,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAS,CAAC;QAClC,MAAM,WAAW,GAAG,MAAM,CAAC,WAAY,CAAC;QAExC,MAAM,eAAe,GAAG,KAAK,EAAE,QAAgB,EAAE,UAAmB,EAAE,EAAE;YACpE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;YAEhD,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACjC,MAAM,IAAI,KAAK,CAAC,aAAa,QAAQ,cAAc,CAAC,CAAC;YACzD,CAAC;QACL,CAAC,CAAC;QACF,MAAM,gBAAgB,GAAG,KAAK,EAAE,QAAiB,EAAE,SAAkB,EAAE,EAAE;YACrE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;YAEhD,IAAI,QAAQ,EAAE,CAAC;gBACX,MAAO,eAA+C,CAAC,eAAe,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;YAChG,CAAC;YAED,OAAO,CAAC,CAAC;QACb,CAAC,CAAC;QAEF,IAAI,CAAC,sBAAsB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAE7C,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC3B,IAAI,CAAC;gBACD,KAAK,MAAM,MAAM,IAAI,sBAAsB,CAAC,QAAQ,CAAC,EAAE,CAAC;oBACpD,IAAI,CAAC,MAAM;wBAAE,SAAS;oBACtB,IAAI,CAAC;wBACD,eAAe,CAAC,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,MAAM,EAAE,QAAQ,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;oBAClG,CAAC;oBAAC,OAAO,CAAC,EAAE,CAAC;wBACT,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,yBAA0B,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;oBACpE,CAAC;gBACL,CAAC;YACL,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAU,EAAE,qCAAqC,CAAC,CAAC;YAC1E,CAAC;QACL,CAAC;QAED,OAAO;YACH,IAAI,IAAI;gBACJ,IAAI,WAAW,CAAC,IAAI,KAAK,0BAA0B;oBAAE,OAAO,IAAI,CAAC;gBACjE,MAAM,UAAU,GAAG,MAAM,CAAC,IAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;gBAC/D,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAClC,CAAC;YACD,eAAe;YACf,gBAAgB;YAChB,WAAW;YACX,IAAI,EAAE,MAAM,CAAC,IAAI;SACpB,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,6BAA6B,CAAC,eAA4C;QACpF,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC;YAC3D,IAAI,KAAK;gBAAE,MAAM,IAAI,YAAY,CAAC,KAAK,CAAC,CAAC;QAC7C,CAAC;QACD,OAAO,EAAE,CAAC;IACd,CAAC;IAES,KAAK,CAAC,gBAAgB,CAAC,eAA4C;QACzE,IAAI,uBAAuB,CAAC,QAAQ,CAAC,eAAe,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;YACrE,MAAM,CAAC,GAAG,MAAM,eAAe,CAAC,gBAAgB,EAAE,CAAC;YAEnD,MAAM,cAAc,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAExF,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,OAAO,oBAAoB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3D,CAAC;QACL,CAAC;QAED,IAAI,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,eAAe,CAAC,QAAQ,CAAC,MAAO,CAAC,EAAE,CAAC;YAChE,OAAO,0BAA0B,eAAe,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;QACvE,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;;OAIG;IACO,KAAK,CAAC,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAA0B;QACnF,MAAM,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;QAEjE,IAAI,CAAC;YACD,OAAO,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACvD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,YAAY,KAAK,IAAI,CAAC,CAAC,WAAW,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;gBAC9D,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;gBACpC,OAAO,IAAI,QAAQ,EAAE,CAAC,CAAC,iEAAiE;YAC5F,CAAC;YAED,IAAI,IAAI,CAAC,YAAY,CAAC,CAAU,CAAC,EAAE,CAAC;gBAChC,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAU,CAAW,CAAC,CAAC;YAC5E,CAAC;iBAAM,CAAC;gBACJ,MAAM,CAAC,CAAC;YACZ,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,cAAc,CAAC,OAAuB,EAAE,QAAkB;QACtE,MAAM,EAAE,MAAM,EAAE,GAAG,QAAQ,CAAC;QAC5B,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,4BAA4B,CAAC,QAAQ,CAAC,CAAC;QACjE,MAAM,EAAE,QAAQ,EAAE,iBAAiB,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;QACnG,MAAM,WAAW,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QAEvC,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;YACjC,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;QAC1C,CAAC;QAED,IAAI,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,IAAI,EAAE,CAAC,CAAC,oGAAoG;YAEjJ,2DAA2D;YAC3D,gDAAgD;YAChD,IAAI,IAAI,KAAK,0BAA0B,EAAE,CAAC;gBACtC,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACvC,IAAI,EAAE,OAAO,EAAE,GAAG,aAAa,CAAC;gBAChC,IAAI,CAAC,OAAO;oBAAE,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,cAAc,EAAE,EAAE,EAAE,CAAC,CAAC;gBACtF,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,MAAM,OAAO,EAAE,CAAC,CAAC;YAC9C,CAAC;YAED,IAAI,IAAI,CAAC,8BAA8B,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBAClD,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,uCAAuC,CAAC,CAAC;YACtE,CAAC;YAED,8EAA8E;YAC9E,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,6BAA6B,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAChF,CAAC;aAAM,IAAI,uBAAuB,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAChD,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC,IAAI,EAAE,EAAE,CAAC;QAC3E,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,iBAAiB,CAAC,KAAK,EAAE,CAAC,CAAC;YAC1D,OAAO;gBACH,IAAI;gBACJ,QAAQ;gBACR,WAAW;aACd,CAAC;QACN,CAAC;IACL,CAAC;IAED;;OAEG;IACO,kBAAkB,CAAC,OAAuB,EAAE,OAAiB,EAAE,QAAiB;QACtF,MAAM,cAAc,GAAG;YACnB,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,QAAQ;YACR,OAAO,EAAE,IAAI,CAAC,uBAAuB;YACrC,YAAY,EAAE,OAAO;YACrB,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,KAAK,EAAE;gBACH,kBAAkB,EAAE,CAAC,IAAI,CAAC,eAAe;aAC5C;YACD,IAAI,EAAE,SAA+B;SACxC,CAAC;QAEF,IAAI,cAAc,CAAC,OAAO,EAAE,MAAM,IAAI,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YACnE,cAAc,CAAC,OAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,2BAA2B,CAAC,OAAO,CAAC,CAAC;YAC3E,OAAO,cAAc,CAAC,OAAQ,CAAC,MAAM,CAAC;QAC1C,CAAC;QAED,4CAA4C;QAC5C,IAAI,OAAO,CAAC,SAAS,EAAE,eAAe,EAAE,CAAC;YACrC,cAAc,CAAC,KAAK,GAAG;gBACnB,GAAG,cAAc,CAAC,KAAK;gBACvB,kBAAkB,EAAE,KAAK;aAC5B,CAAC;QACN,CAAC;QAED,IAAI,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,cAAc,CAAC,IAAI,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;QAEvF,OAAO,cAAc,CAAC;IAC1B,CAAC;IAES,eAAe,CACrB,OAAuB,EACvB,QAAkB,EAClB,QAAwB;QAKxB,IAAI,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAC7B,QAAQ,GAAG,IAAI,CAAC,qBAAuC,CAAC;QAC5D,CAAC;aAAM,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,uBAAuB,EAAE,CAAC;YACnD,QAAQ,GAAG,IAAI,CAAC,uBAAyC,CAAC;QAC9D,CAAC;QAED,sDAAsD;QACtD,MAAM,IAAI,GAAG,MAAM,CAAC;QACpB,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAEnD,2DAA2D;QAC3D,+CAA+C;QAC/C,IAAI,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;QAE/D,+DAA+D;QAC/D,IAAI,KAAK,CAAC,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC;YACjC,MAAM,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,YAAY,GAAG,KAAK;iBACrB,YAAY,CAAC,QAAQ,CAAC;iBACtB,EAAE,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YAClE,MAAM,aAAa,GAAG,QAAQ,CAAC,IAAI;gBAC/B,CAAC,CAAC,QAAQ,CAAC,KAAK,CACV,QAAQ,CAAC,IAAI,CACT,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAW,CAAC;qBACjC,IAAI,CAAC,YAAY,CAAC;qBAClB,IAAI,CAAC,YAAY,CAAC,CAC1B,CACJ;gBACH,CAAC,CAAC,IAAI,CAAC;YAEX,OAAO;gBACH,QAAQ,EAAE,IAAI,eAAe,CAAC,aAAoB,EAAE,QAAQ,CAAC;gBAC7D,QAAQ,EAAE,IAAI;aACjB,CAAC;QACN,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,YAAY,OAAO,CAAC,GAAG,8CAA8C,QAAQ,EAAE,CAAC,CAAC;IACrG,CAAC;IAED;;OAEG;IACO,yBAAyB,CAAC,mBAA4D;QAC5F,KAAK,MAAM,QAAQ,IAAI,mBAAmB,EAAE,CAAC;YACzC,IAAI,QAAQ,KAAK,KAAK,EAAE,CAAC;gBACrB,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBACtC,SAAS;YACb,CAAC;YAED,IAAI,CAAC;gBACD,MAAM,UAAU,GAAG,iBAAiB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBACrD,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;YACjD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,IAAI,KAAK,CAAC,2BAA2B,QAAQ,sCAAsC,CAAC,CAAC;YAC/F,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACO,qBAAqB,CAAC,OAAiB;QAC7C,OAAO,CAAC,OAAO,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,uBAAuB,GAAG,IAAI,WAAW,CAAC,CAAC;IAC/F,CAAC;IAEO,oBAAoB,CAAC,OAAuB,EAAE,QAAkB;QACpE,MAAM,EAAE,MAAM,EAAE,GAAG,QAAQ,CAAC;QAC5B,MAAM,EAAE,IAAI,EAAE,GAAG,4BAA4B,CAAC,QAAQ,CAAC,CAAC;QAExD,MAAM,sBAAsB,GAAG,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAEpF,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;YACvG,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;YACvB,MAAM,IAAI,KAAK,CACX,YAAY,OAAO,CAAC,GAAG,wBAAwB,IAAI,IAAI;gBACnD,YAAY,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,kCAAkC,CACnG,CAAC;QACN,CAAC;IACL,CAAC;IAED;;OAEG;IACK,iBAAiB,GAAG,KAAK,EAAE,OAAwB,EAAE,OAAiB,EAAE,EAAE;QAC9E,MAAM,IAAI,GAAG,yBAAyB,CAAC;YACnC,GAAI,OAAe;YACnB,YAAY,EAAE,MAAM;SACvB,CAAC,CAAC;QAEH,8EAA8E;QAC9E,qFAAqF;QACrF,qFAAqF;QACrF,MAAM,SAAS,GAAG,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,OAAO,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QAEjG,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,CAC9C,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE;YAClB,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAS,CAAC,CAAC,CAAC,SAAS;YAChE,OAAO,EAAE,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;YAClC,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,kEAAkE;YAClE,MAAM,EAAE,MAAM;SACF,CAAC,EACjB;YACI,OAAO;YACP,SAAS;YACT,aAAa,EAAE,IAAI,CAAC,OAAO;SAC9B,CACJ,CAAC;QAEF,OAAO,QAAQ,CAAC;IACpB,CAAC,CAAC;;AASN;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,UAAU,gBAAgB,CAG9B,MAAwC;IACtC,OAAO,MAAM,CAAC,MAAM,CAAU,MAAM,CAAC,CAAC;AAC1C,CAAC"}
1
+ {"version":3,"file":"http-crawler.js","sourceRoot":"","sources":["../../src/internals/http-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,IAAI,MAAM,WAAW,CAAC;AAc7B,OAAO,EACH,YAAY,EACZ,eAAe,EACf,sBAAsB,EACtB,YAAY,EACZ,MAAM,EACN,YAAY,GACf,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAsB,sBAAsB,EAAE,MAAM,eAAe,CAAC;AAC3E,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAEvD,OAAO,EAAoB,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AACvE,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,OAAO,iBAAiB,MAAM,cAAc,CAAC;AAC7C,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,EAAE,MAAM,IAAI,CAAC;AAGpB,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAEhE,OAAO,EAAE,2BAA2B,EAAE,4BAA4B,EAAE,yBAAyB,EAAE,MAAM,YAAY,CAAC;AAElH;;GAEG;AACH,MAAM,uBAAuB,GAAG,CAAC,WAAW,EAAE,UAAU,EAAE,uBAAuB,EAAE,iBAAiB,CAAC,CAAC;AACtG,MAAM,0BAA0B,GAAG,kBAAkB,CAAC;AACtD,MAAM,sCAAsC,GAA0B;IAClE,kBAAkB,EAAE,EAAE;IACtB,kBAAkB,EAAE;QAChB,6BAA6B,EAAE,CAAC;QAChC,gBAAgB,EAAE,GAAG;KACxB;IACD,mBAAmB,EAAE;QACjB,2BAA2B,EAAE,GAAG;KACnC;CACJ,CAAC;AA6LF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoEG;AACH,MAAM,OAAO,WAIX,SAAQ,YAAwD;IACpD,kBAAkB,CAAsC;IACxD,mBAAmB,CAEkC;IACrD,mBAAmB,CAAU;IAC7B,uBAAuB,CAAS;IAChC,eAAe,CAAU;IACzB,uBAAuB,CAAU;IACjC,qBAAqB,CAAU;IACtB,kBAAkB,CAAc;IAEzC,MAAM,CAAU,YAAY,GAAG;QACrC,GAAG,YAAY,CAAC,YAAY;QAE5B,qBAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QACpC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;QACxD,uBAAuB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC3C,qBAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QAExC,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK;QACrC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK;KACzC,CAAC;IAEF;;OAEG;IACH,YACI,UACmE,EAAS;QAE5E,EAAE,CAAC,OAAO,EAAE,oBAAoB,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC;QAElF,MAAM,EACF,qBAAqB,GAAG,EAAE,EAC1B,eAAe,GAAG,IAAI,EACtB,mBAAmB,GAAG,EAAE,EACxB,uBAAuB,EACvB,qBAAqB,EACrB,mBAAmB,GAAG,IAAI,EAC1B,kBAAkB,GAAG,EAAE,EACvB,mBAAmB,GAAG,EAAE;QAExB,eAAe;QACf,qBAAqB,GAAG,sCAAsC,EAC9D,sBAAsB,EACtB,GAAG,mBAAmB,EACzB,GAAG,OAAO,CAAC;QAEZ,KAAK,CAAC;YACF,GAAG,mBAAmB;YACtB,qBAAqB;YACrB,sBAAsB,EAClB,sBAAsB;gBACtB,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,oBAAoB,EAA+C,CAAC;SACvF,CAAC,CAAC;QAEH,IAAI,CAAC,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,uBAAuB,EAAE,0BAA0B,CAAC,CAAC,CAAC;QAC5F,IAAI,mBAAmB,CAAC,MAAM;YAAE,IAAI,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC;QAEpF,IAAI,uBAAuB,IAAI,qBAAqB,EAAE,CAAC;YACnD,IAAI,CAAC,GAAG,CAAC,OAAO,CACZ,sGAAsG,CACzG,CAAC;QACN,CAAC;QAED,IAAI,CAAC,uBAAuB,GAAG,qBAAqB,GAAG,IAAI,CAAC;QAC5D,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,uBAAuB,GAAG,uBAAuB,CAAC;QACvD,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,CAAC;QACnD,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,mBAAmB,GAAG;YACvB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,oBAAoB,CAAC,OAAO,EAAE,QAAS,CAAC;YACxE,GAAG,mBAAmB;SACzB,CAAC;QAEF,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;IACnD,CAAC;IAEkB,oBAAoB;QACnC,8FAA8F;QAC9F,kGAAkG;QAClG,MAAM,SAAS,GAAG,CACd,MAA2C,EAChB,EAAE,CAAC,CAAC;YAC/B,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAQ;SAChG,CAAC,CAAC;QAEH,IAAI,QAAQ,GAAG,eAAe,CAAC,MAAM,EAAmB,CAAC,OAAO,CAAC;YAC7D,MAAM,EAAE,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC;SAC7C,CAAC,CAAC;QAEH,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACzC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;QACjD,CAAC;QAED,IAAI,sBAAsB,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAE1F,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC1C,sBAAsB,GAAG,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;QAC7E,CAAC;QAED,OAAO,sBAAsB;aACxB,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;aACxD,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,6BAA6B,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC5E,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,eAAgC;QAC7D,MAAM,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAEpC,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YACzB,OAAO;gBACH,OAAO,EAAE,IAAI,KAAK,CAAC,OAAO,EAAE;oBACxB,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ;wBAC9B,IAAI,YAAY,KAAK,WAAW,EAAE,CAAC;4BAC/B,MAAM,IAAI,sBAAsB,CAC5B,+EAA+E,CAClF,CAAC;wBACN,CAAC;wBACD,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ,CAAC,CAAC;oBACvD,CAAC;iBACJ,CAAkC;gBACnC,IAAI,QAAQ;oBACR,MAAM,IAAI,sBAAsB,CAC5B,sEAAsE,CACzE,CAAC;gBACN,CAAC;aACoC,CAAC;QAC9C,CAAC;QAED,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,UAAU,CAAC;QACxC,OAAO,EAAE,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,eAAe,CACzB,eAAgC;QAEhC,SAAS,EAAE,CAAC;QAEZ,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAC7C,MAAM,QAAQ,GAAG,eAAe,CAAC,SAAS,EAAE,GAAG,CAAC;QAEhD,MAAM,YAAY,GAAG,MAAM,mBAAmB,CAC1C,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,EACjE,IAAI,CAAC,uBAAuB,EAC5B,2BAA2B,IAAI,CAAC,uBAAuB,GAAG,IAAI,WAAW,CAC5E,CAAC;QACF,SAAS,EAAE,CAAC;QAEZ,OAAO,CAAC,SAAS,GAAG,YAAY,EAAE,GAAG,CAAC;QACtC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,SAAS,CAAC;QAEvC,OAAO,EAAE,OAAO,EAAE,OAAwC,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC;IACzF,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAC7B,eAA4C;QAI5C,IAAI,eAAe,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YACzC,OAAO;gBACH,IAAI,WAAW;oBACX,MAAM,IAAI,sBAAsB,CAC5B,yEAAyE,CAC5E,CAAC;gBACN,CAAC;gBACD,IAAI,IAAI;oBACJ,MAAM,IAAI,sBAAsB,CAC5B,kEAAkE,CACrE,CAAC;gBACN,CAAC;gBACD,IAAI,IAAI;oBACJ,MAAM,IAAI,sBAAsB,CAC5B,kEAAkE,CACrE,CAAC;gBACN,CAAC;gBACD,IAAI,eAAe;oBACf,MAAM,IAAI,sBAAsB,CAC5B,2EAA2E,CAC9E,CAAC;gBACN,CAAC;gBACD,IAAI,gBAAgB;oBAChB,MAAM,IAAI,sBAAsB,CAC5B,4EAA4E,CAC/E,CAAC;gBACN,CAAC;aACJ,CAAC;QACN,CAAC;QAED,SAAS,EAAE,CAAC;QAEZ,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,OAAO,EAAE,eAAe,CAAC,QAAQ,CAAC,CAAC;QAC5F,SAAS,EAAE,CAAC;QACZ,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAS,CAAC;QAClC,MAAM,WAAW,GAAG,MAAM,CAAC,WAAY,CAAC;QAExC,MAAM,eAAe,GAAG,KAAK,EAAE,QAAgB,EAAE,UAAmB,EAAE,EAAE;YACpE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;YAEhD,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACjC,MAAM,IAAI,KAAK,CAAC,aAAa,QAAQ,cAAc,CAAC,CAAC;YACzD,CAAC;QACL,CAAC,CAAC;QACF,MAAM,gBAAgB,GAAG,KAAK,EAAE,QAAiB,EAAE,SAAkB,EAAE,EAAE;YACrE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;YAEhD,IAAI,QAAQ,EAAE,CAAC;gBACX,MAAO,eAA+C,CAAC,eAAe,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;YAChG,CAAC;YAED,OAAO,CAAC,CAAC;QACb,CAAC,CAAC;QAEF,IAAI,CAAC,sBAAsB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAE7C,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC3B,IAAI,CAAC;gBACD,KAAK,MAAM,MAAM,IAAI,sBAAsB,CAAC,QAAQ,CAAC,EAAE,CAAC;oBACpD,IAAI,CAAC,MAAM;wBAAE,SAAS;oBACtB,IAAI,CAAC;wBACD,eAAe,CAAC,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,MAAM,EAAE,QAAQ,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;oBAClG,CAAC;oBAAC,OAAO,CAAC,EAAE,CAAC;wBACT,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,yBAA0B,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;oBACpE,CAAC;gBACL,CAAC;YACL,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAU,EAAE,qCAAqC,CAAC,CAAC;YAC1E,CAAC;QACL,CAAC;QAED,OAAO;YACH,IAAI,IAAI;gBACJ,IAAI,WAAW,CAAC,IAAI,KAAK,0BAA0B;oBAAE,OAAO,IAAI,CAAC;gBACjE,MAAM,UAAU,GAAG,MAAM,CAAC,IAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;gBAC/D,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAClC,CAAC;YACD,eAAe;YACf,gBAAgB;YAChB,WAAW;YACX,IAAI,EAAE,MAAM,CAAC,IAAI;SACpB,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,6BAA6B,CAAC,eAA4C;QACpF,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC;YAC3D,IAAI,KAAK;gBAAE,MAAM,IAAI,YAAY,CAAC,KAAK,CAAC,CAAC;QAC7C,CAAC;QACD,OAAO,EAAE,CAAC;IACd,CAAC;IAES,KAAK,CAAC,gBAAgB,CAAC,eAA4C;QACzE,IAAI,uBAAuB,CAAC,QAAQ,CAAC,eAAe,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;YACrE,MAAM,CAAC,GAAG,MAAM,eAAe,CAAC,gBAAgB,EAAE,CAAC;YAEnD,MAAM,cAAc,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAExF,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,OAAO,oBAAoB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3D,CAAC;QACL,CAAC;QAED,IAAI,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,eAAe,CAAC,QAAQ,CAAC,MAAO,CAAC,EAAE,CAAC;YAChE,OAAO,0BAA0B,eAAe,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;QACvE,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;;OAIG;IACO,KAAK,CAAC,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAA0B;QACnF,MAAM,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;QAEjE,IAAI,CAAC;YACD,OAAO,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACvD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,YAAY,KAAK,IAAI,CAAC,CAAC,WAAW,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;gBAC9D,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;gBACpC,OAAO,IAAI,QAAQ,EAAE,CAAC,CAAC,iEAAiE;YAC5F,CAAC;YAED,IAAI,IAAI,CAAC,YAAY,CAAC,CAAU,CAAC,EAAE,CAAC;gBAChC,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAU,CAAW,CAAC,CAAC;YAC5E,CAAC;iBAAM,CAAC;gBACJ,MAAM,CAAC,CAAC;YACZ,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,cAAc,CAAC,OAAuB,EAAE,QAAkB;QACtE,MAAM,EAAE,MAAM,EAAE,GAAG,QAAQ,CAAC;QAC5B,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,4BAA4B,CAAC,QAAQ,CAAC,CAAC;QACjE,MAAM,EAAE,QAAQ,EAAE,iBAAiB,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;QACnG,MAAM,WAAW,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QAEvC,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;YACjC,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;QAC1C,CAAC;QAED,IAAI,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,IAAI,EAAE,CAAC,CAAC,oGAAoG;YAEjJ,2DAA2D;YAC3D,gDAAgD;YAChD,IAAI,IAAI,KAAK,0BAA0B,EAAE,CAAC;gBACtC,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACvC,IAAI,EAAE,OAAO,EAAE,GAAG,aAAa,CAAC;gBAChC,IAAI,CAAC,OAAO;oBAAE,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,cAAc,EAAE,EAAE,EAAE,CAAC,CAAC;gBACtF,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,MAAM,OAAO,EAAE,CAAC,CAAC;YAC9C,CAAC;YAED,IAAI,IAAI,CAAC,8BAA8B,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBAClD,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,uCAAuC,CAAC,CAAC;YACtE,CAAC;YAED,8EAA8E;YAC9E,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,6BAA6B,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAChF,CAAC;aAAM,IAAI,uBAAuB,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAChD,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,qBAAqB,EAAE,CAAC;gBAC1C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;gBAC3D,MAAM,WAAW,GAAG,2BAA2B,CAAC,QAAQ,CAAC,CAAC;gBAC1D,MAAM,YAAY,GAAG,WAAW,IAAI,IAAI,CAAC,uBAAuB,IAAI,OAAO,CAAC;gBAC5E,MAAM,IAAI,GAAG,KAAK,CAAC,cAAc,CAAC,YAAY,CAAC;oBAC3C,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE,YAAY,CAAC;oBACtC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;gBAChC,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAyB,EAAE,EAAE,IAAI,EAAE,CAAC;YAC1F,CAAC;YACD,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC,IAAI,EAAE,EAAE,CAAC;QAC3E,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,iBAAiB,CAAC,KAAK,EAAE,CAAC,CAAC;YAC1D,OAAO;gBACH,IAAI;gBACJ,QAAQ;gBACR,WAAW;aACd,CAAC;QACN,CAAC;IACL,CAAC;IAED;;OAEG;IACO,kBAAkB,CAAC,OAAuB,EAAE,OAAiB,EAAE,QAAiB;QACtF,MAAM,cAAc,GAAG;YACnB,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,QAAQ;YACR,OAAO,EAAE,IAAI,CAAC,uBAAuB;YACrC,YAAY,EAAE,OAAO;YACrB,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,KAAK,EAAE;gBACH,kBAAkB,EAAE,CAAC,IAAI,CAAC,eAAe;aAC5C;YACD,IAAI,EAAE,SAA+B;SACxC,CAAC;QAEF,IAAI,cAAc,CAAC,OAAO,EAAE,MAAM,IAAI,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YACnE,cAAc,CAAC,OAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,2BAA2B,CAAC,OAAO,CAAC,CAAC;YAC3E,OAAO,cAAc,CAAC,OAAQ,CAAC,MAAM,CAAC;QAC1C,CAAC;QAED,4CAA4C;QAC5C,IAAI,OAAO,CAAC,SAAS,EAAE,eAAe,EAAE,CAAC;YACrC,cAAc,CAAC,KAAK,GAAG;gBACnB,GAAG,cAAc,CAAC,KAAK;gBACvB,kBAAkB,EAAE,KAAK;aAC5B,CAAC;QACN,CAAC;QAED,IAAI,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,cAAc,CAAC,IAAI,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;QAEvF,OAAO,cAAc,CAAC;IAC1B,CAAC;IAES,eAAe,CACrB,OAAuB,EACvB,QAAkB,EAClB,QAAwB;QAKxB,IAAI,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAC7B,QAAQ,GAAG,IAAI,CAAC,qBAAuC,CAAC;QAC5D,CAAC;aAAM,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,uBAAuB,EAAE,CAAC;YACnD,QAAQ,GAAG,IAAI,CAAC,uBAAyC,CAAC;QAC9D,CAAC;QAED,sDAAsD;QACtD,MAAM,IAAI,GAAG,MAAM,CAAC;QACpB,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAEnD,2DAA2D;QAC3D,+CAA+C;QAC/C,IAAI,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;QAE/D,+DAA+D;QAC/D,IAAI,KAAK,CAAC,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC;YACjC,MAAM,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,YAAY,GAAG,KAAK;iBACrB,YAAY,CAAC,QAAQ,CAAC;iBACtB,EAAE,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YAClE,MAAM,aAAa,GAAG,QAAQ,CAAC,IAAI;gBAC/B,CAAC,CAAC,QAAQ,CAAC,KAAK,CACV,QAAQ,CAAC,IAAI,CACT,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAW,CAAC;qBACjC,IAAI,CAAC,YAAY,CAAC;qBAClB,IAAI,CAAC,YAAY,CAAC,CAC1B,CACJ;gBACH,CAAC,CAAC,IAAI,CAAC;YAEX,OAAO;gBACH,QAAQ,EAAE,IAAI,eAAe,CAAC,aAAoB,EAAE,QAAQ,CAAC;gBAC7D,QAAQ,EAAE,IAAI;aACjB,CAAC;QACN,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,YAAY,OAAO,CAAC,GAAG,8CAA8C,QAAQ,EAAE,CAAC,CAAC;IACrG,CAAC;IAED;;OAEG;IACO,yBAAyB,CAAC,mBAA4D;QAC5F,KAAK,MAAM,QAAQ,IAAI,mBAAmB,EAAE,CAAC;YACzC,IAAI,QAAQ,KAAK,KAAK,EAAE,CAAC;gBACrB,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBACtC,SAAS;YACb,CAAC;YAED,IAAI,CAAC;gBACD,MAAM,UAAU,GAAG,iBAAiB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBACrD,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;YACjD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,IAAI,KAAK,CAAC,2BAA2B,QAAQ,sCAAsC,CAAC,CAAC;YAC/F,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACO,qBAAqB,CAAC,OAAiB;QAC7C,OAAO,CAAC,OAAO,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,uBAAuB,GAAG,IAAI,WAAW,CAAC,CAAC;IAC/F,CAAC;IAEO,oBAAoB,CAAC,OAAuB,EAAE,QAAkB;QACpE,MAAM,EAAE,MAAM,EAAE,GAAG,QAAQ,CAAC;QAC5B,MAAM,EAAE,IAAI,EAAE,GAAG,4BAA4B,CAAC,QAAQ,CAAC,CAAC;QAExD,MAAM,sBAAsB,GAAG,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAEpF,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;YACvG,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;YACvB,MAAM,IAAI,KAAK,CACX,YAAY,OAAO,CAAC,GAAG,wBAAwB,IAAI,IAAI;gBACnD,YAAY,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,kCAAkC,CACnG,CAAC;QACN,CAAC;IACL,CAAC;IAED;;OAEG;IACK,iBAAiB,GAAG,KAAK,EAAE,OAAwB,EAAE,OAAiB,EAAE,EAAE;QAC9E,MAAM,IAAI,GAAG,yBAAyB,CAAC;YACnC,GAAI,OAAe;YACnB,YAAY,EAAE,MAAM;SACvB,CAAC,CAAC;QAEH,8EAA8E;QAC9E,qFAAqF;QACrF,qFAAqF;QACrF,MAAM,SAAS,GAAG,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,OAAO,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QAEjG,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,CAC9C,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE;YAClB,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAS,CAAC,CAAC,CAAC,SAAS;YAChE,OAAO,EAAE,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;YAClC,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,kEAAkE;YAClE,MAAM,EAAE,MAAM;SACF,CAAC,EACjB;YACI,OAAO;YACP,SAAS;YACT,aAAa,EAAE,IAAI,CAAC,OAAO;SAC9B,CACJ,CAAC;QAEF,OAAO,QAAQ,CAAC;IACpB,CAAC,CAAC;;AASN;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,UAAU,gBAAgB,CAG9B,MAAwC;IACtC,OAAO,MAAM,CAAC,MAAM,CAAU,MAAM,CAAC,CAAC;AAC1C,CAAC"}
package/internals/utils.d.ts CHANGED
@@ -3,6 +3,12 @@ import type { HttpRequest, HttpRequestOptions } from '@crawlee/types';
3
3
  * Converts {@link HttpRequestOptions} to a {@link HttpRequest}.
4
4
  */
5
5
  export declare function processHttpRequestOptions({ searchParams, form, json, username, password, ...request }: HttpRequestOptions): HttpRequest;
6
+ /**
7
+ * Scans the first 1024 bytes of an HTML document (as latin1) to extract the charset
8
+ * declared via `<meta charset>` or `<meta http-equiv="Content-Type" content="...;charset=...">`.
9
+ * This implements a simplified version of the HTML spec's byte-stream prescan algorithm.
10
+ */
11
+ export declare function extractCharsetFromHtmlBytes(bytes: Buffer): string | undefined;
6
12
  /**
7
13
  * Gets parsed content type from response object
8
14
  * @param response HTTP response object
package/internals/utils.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/internals/utils.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,WAAW,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAMtE;;GAEG;AACH,wBAAgB,yBAAyB,CAAC,EACtC,YAAY,EACZ,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,GAAG,OAAO,EACb,EAAE,kBAAkB,GAAG,WAAW,CAwClC;AAED;;;GAGG;AACH,wBAAgB,4BAA4B,CAAC,QAAQ,EAAE,QAAQ,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,cAAc,CAAA;CAAE,CAgC1G"}
1
+ {"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/internals/utils.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,WAAW,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAMtE;;GAEG;AACH,wBAAgB,yBAAyB,CAAC,EACtC,YAAY,EACZ,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,GAAG,OAAO,EACb,EAAE,kBAAkB,GAAG,WAAW,CAwClC;AAED;;;;GAIG;AACH,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAK7E;AAED;;;GAGG;AACH,wBAAgB,4BAA4B,CAAC,QAAQ,EAAE,QAAQ,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,cAAc,CAAA;CAAE,CAgC1G"}
package/internals/utils.js CHANGED
@@ -38,6 +38,17 @@ export function processHttpRequestOptions({ searchParams, form, json, username,
38
38
  }
39
39
  return { ...request, body, url, headers };
40
40
  }
41
+ /**
42
+ * Scans the first 1024 bytes of an HTML document (as latin1) to extract the charset
43
+ * declared via `<meta charset>` or `<meta http-equiv="Content-Type" content="...;charset=...">`.
44
+ * This implements a simplified version of the HTML spec's byte-stream prescan algorithm.
45
+ */
46
+ export function extractCharsetFromHtmlBytes(bytes) {
47
+ // latin1 preserves byte values for ASCII-compatible encodings, making the meta tags readable
48
+ const prescan = bytes.subarray(0, 1024).toString('latin1');
49
+ const match = /<meta[^>]+\bcharset\s*=\s*["']?\s*([^"'\s;>]+)/i.exec(prescan);
50
+ return match?.[1];
51
+ }
41
52
  /**
42
53
  * Gets parsed content type from response object
43
54
  * @param response HTTP response object
package/internals/utils.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/internals/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAGvC,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,iBAAiB,MAAM,cAAc,CAAC;AAC7C,OAAO,IAAI,MAAM,YAAY,CAAC;AAC9B,OAAO,EAAE,EAAE,EAAE,eAAe,EAAE,MAAM,IAAI,CAAC;AAEzC;;GAEG;AACH,MAAM,UAAU,yBAAyB,CAAC,EACtC,YAAY,EACZ,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,GAAG,OAAO,EACO;IACjB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACjC,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAE7C,iBAAiB,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IAErC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/E,MAAM,IAAI,KAAK,CAAC,oFAAoF,CAAC,CAAC;IAC1G,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,GAAG,EAAE;QACf,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACrB,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC/D,CAAC;QAED,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACrB,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/C,CAAC;QAED,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC7B,OAAO,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACvC,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC,CAAC,EAAE,CAAC;IAEL,IAAI,IAAI,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,mCAAmC,CAAC,CAAC;IACrE,CAAC;IAED,IAAI,IAAI,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,kBAAkB,CAAC,CAAC;IACpD,CAAC;IAED,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QACnD,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,IAAI,EAAE,IAAI,QAAQ,IAAI,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC1F,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,SAAS,WAAW,EAAE,CAAC,CAAC;IACzD,CAAC;IAED,OAAO,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC;AAC9C,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,4BAA4B,CAAC,QAAkB;IAC3D,EAAE,CACE,QAAQ,EACR,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC;QACnB,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG;QAClB,OAAO,EAAE,IAAI,eAAe,EAA2B;KAC1D,CAAC,CACL,CAAC;IAEF,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAC;IAClC,IAAI,iBAAiB,CAAC;IAEtB,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;QAC9B,IAAI,CAAC;YACD,iBAAiB,GAAG,iBAAiB,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAW,CAAC,CAAC;QACvF,CAAC;QAAC,MAAM,CAAC;YACL,4FAA4F;QAChG,CAAC;IACL,CAAC;IAED,qDAAqD;IACrD,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACrB,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/B,MAAM,sBAAsB,GACxB,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,IAAI,yCAAyC,CAAC,CAAC,0FAA0F;QAC1L,iBAAiB,GAAG,iBAAiB,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;IACxE,CAAC;IAED,OAAO;QACH,IAAI,EAAE,iBAAiB,CAAC,IAAI;QAC5B,OAAO,EAAE,iBAAiB,CAAC,UAAU,CAAC,OAAyB;KAClE,CAAC;AACN,CAAC"}
1
+ {"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/internals/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAGvC,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,iBAAiB,MAAM,cAAc,CAAC;AAC7C,OAAO,IAAI,MAAM,YAAY,CAAC;AAC9B,OAAO,EAAE,EAAE,EAAE,eAAe,EAAE,MAAM,IAAI,CAAC;AAEzC;;GAEG;AACH,MAAM,UAAU,yBAAyB,CAAC,EACtC,YAAY,EACZ,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,GAAG,OAAO,EACO;IACjB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACjC,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAE7C,iBAAiB,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IAErC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/E,MAAM,IAAI,KAAK,CAAC,oFAAoF,CAAC,CAAC;IAC1G,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,GAAG,EAAE;QACf,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACrB,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC/D,CAAC;QAED,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACrB,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/C,CAAC;QAED,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC7B,OAAO,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACvC,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC,CAAC,EAAE,CAAC;IAEL,IAAI,IAAI,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,mCAAmC,CAAC,CAAC;IACrE,CAAC;IAED,IAAI,IAAI,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,kBAAkB,CAAC,CAAC;IACpD,CAAC;IAED,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QACnD,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,IAAI,EAAE,IAAI,QAAQ,IAAI,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC1F,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,SAAS,WAAW,EAAE,CAAC,CAAC;IACzD,CAAC;IAED,OAAO,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC;AAC9C,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,2BAA2B,CAAC,KAAa;IACrD,6FAA6F;IAC7F,MAAM,OAAO,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAC3D,MAAM,KAAK,GAAG,iDAAiD,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC9E,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;AACtB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,4BAA4B,CAAC,QAAkB;IAC3D,EAAE,CACE,QAAQ,EACR,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC;QACnB,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG;QAClB,OAAO,EAAE,IAAI,eAAe,EAA2B;KAC1D,CAAC,CACL,CAAC;IAEF,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAC;IAClC,IAAI,iBAAiB,CAAC;IAEtB,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;QAC9B,IAAI,CAAC;YACD,iBAAiB,GAAG,iBAAiB,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAW,CAAC,CAAC;QACvF,CAAC;QAAC,MAAM,CAAC;YACL,4FAA4F;QAChG,CAAC;IACL,CAAC;IAED,qDAAqD;IACrD,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACrB,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/B,MAAM,sBAAsB,GACxB,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,IAAI,yCAAyC,CAAC,CAAC,0FAA0F;QAC1L,iBAAiB,GAAG,iBAAiB,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;IACxE,CAAC;IAED,OAAO;QACH,IAAI,EAAE,iBAAiB,CAAC,IAAI;QAC5B,OAAO,EAAE,iBAAiB,CAAC,UAAU,CAAC,OAAyB;KAClE,CAAC;AACN,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/http",
3
- "version": "4.0.0-beta.66",
3
+ "version": "4.0.0-beta.67",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
6
  "node": ">=22.0.0"
@@ -49,11 +49,11 @@
49
49
  "dependencies": {
50
50
  "@apify/timeout": "^0.3.2",
51
51
  "@apify/utilities": "^2.15.5",
52
- "@crawlee/basic": "4.0.0-beta.66",
53
- "@crawlee/core": "4.0.0-beta.66",
54
- "@crawlee/http-client": "4.0.0-beta.66",
55
- "@crawlee/types": "4.0.0-beta.66",
56
- "@crawlee/utils": "4.0.0-beta.66",
52
+ "@crawlee/basic": "4.0.0-beta.67",
53
+ "@crawlee/core": "4.0.0-beta.67",
54
+ "@crawlee/http-client": "4.0.0-beta.67",
55
+ "@crawlee/types": "4.0.0-beta.67",
56
+ "@crawlee/utils": "4.0.0-beta.67",
57
57
  "@types/content-type": "^1.1.8",
58
58
  "cheerio": "^1.0.0",
59
59
  "content-type": "^1.0.5",
@@ -70,5 +70,5 @@
70
70
  }
71
71
  }
72
72
  },
73
- "gitHead": "0890633b51caf16d93940be6f7da7881ad839992"
73
+ "gitHead": "20617e873491494f343cd32831645acec484abed"
74
74
  }