@crawlee/utils 4.0.0-beta.2 → 4.0.0-beta.21

package/README.md

@@ -9,6 +9,10 @@
     <small>A web scraping and browser automation library</small>
 </h1>
 
+<p align=center>
+    <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
+</p>
+
 <p align=center>
     <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" style="max-width: 100%;"></a>
     <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" style="max-width: 100%;"></a>
@@ -24,7 +28,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
 
 > 👉 **View full documentation, guides and examples on the [Crawlee project website](https://crawlee.dev)** 👈
 
-> Crawlee for Python is open for early adopters. 🐍  [👉 Checkout the source code 👈](https://github.com/apify/crawlee-python).
+> Do you prefer 🐍 Python instead of JavaScript? [👉 Checkout Crawlee for Python 👈](https://github.com/apify/crawlee-python).
 
 ## Installation
 

package/index.d.ts

@@ -3,15 +3,17 @@ export * from './internals/cheerio.js';
 export * from './internals/chunk.js';
 export * from './internals/extract-urls.js';
 export * from './internals/general.js';
-export * from './internals/memory-info.js';
 export * from './internals/debug.js';
 export * as social from './internals/social.js';
 export * from './internals/typedefs.js';
 export * from './internals/open_graph_parser.js';
 export * from './internals/robots.js';
 export * from './internals/sitemap.js';
+export * from './internals/iterables.js';
+export * from './internals/robots.js';
+export * from './internals/sitemap.js';
 export * from './internals/url.js';
-export { getCurrentCpuTicksV2 } from './internals/systemInfoV2/cpu-info.js';
-export { getMemoryInfoV2 } from './internals/systemInfoV2/memory-info.js';
+export { getCurrentCpuTicksV2, CpuSample } from './internals/system-info/cpu-info.js';
+export { getMemoryInfo, MemoryInfo } from './internals/system-info/memory-info.js';
 export { Dictionary, Awaitable, Constructor } from '@crawlee/types';
 //# sourceMappingURL=index.d.ts.map

package/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,wBAAwB,CAAC;AACvC,cAAc,wBAAwB,CAAC;AACvC,cAAc,sBAAsB,CAAC;AACrC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,wBAAwB,CAAC;AACvC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,sBAAsB,CAAC;AACrC,OAAO,KAAK,MAAM,MAAM,uBAAuB,CAAC;AAChD,cAAc,yBAAyB,CAAC;AACxC,cAAc,kCAAkC,CAAC;AACjD,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC;AAEnC,OAAO,EAAE,oBAAoB,EAAE,MAAM,sCAAsC,CAAC;AAC5E,OAAO,EAAE,eAAe,EAAE,MAAM,yCAAyC,CAAC;AAE1E,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,wBAAwB,CAAC;AACvC,cAAc,wBAAwB,CAAC;AACvC,cAAc,sBAAsB,CAAC;AACrC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,wBAAwB,CAAC;AACvC,cAAc,sBAAsB,CAAC;AACrC,OAAO,KAAK,MAAM,MAAM,uBAAuB,CAAC;AAChD,cAAc,yBAAyB,CAAC;AACxC,cAAc,kCAAkC,CAAC;AACjD,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC;AACvC,cAAc,0BAA0B,CAAC;AACzC,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC;AAEnC,OAAO,EAAE,oBAAoB,EAAE,SAAS,EAAE,MAAM,qCAAqC,CAAC;AACtF,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,wCAAwC,CAAC;AAEnF,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC"}

package/index.js

@@ -3,14 +3,16 @@ export * from './internals/cheerio.js';
 export * from './internals/chunk.js';
 export * from './internals/extract-urls.js';
 export * from './internals/general.js';
-export * from './internals/memory-info.js';
 export * from './internals/debug.js';
 export * as social from './internals/social.js';
 export * from './internals/typedefs.js';
 export * from './internals/open_graph_parser.js';
 export * from './internals/robots.js';
 export * from './internals/sitemap.js';
+export * from './internals/iterables.js';
+export * from './internals/robots.js';
+export * from './internals/sitemap.js';
 export * from './internals/url.js';
-export { getCurrentCpuTicksV2 } from './internals/systemInfoV2/cpu-info.js';
-export { getMemoryInfoV2 } from './internals/systemInfoV2/memory-info.js';
+export { getCurrentCpuTicksV2 } from './internals/system-info/cpu-info.js';
+export { getMemoryInfo } from './internals/system-info/memory-info.js';
 //# sourceMappingURL=index.js.map

package/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,wBAAwB,CAAC;AACvC,cAAc,wBAAwB,CAAC;AACvC,cAAc,sBAAsB,CAAC;AACrC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,wBAAwB,CAAC;AACvC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,sBAAsB,CAAC;AACrC,OAAO,KAAK,MAAM,MAAM,uBAAuB,CAAC;AAChD,cAAc,yBAAyB,CAAC;AACxC,cAAc,kCAAkC,CAAC;AACjD,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC;AAEnC,OAAO,EAAE,oBAAoB,EAAE,MAAM,sCAAsC,CAAC;AAC5E,OAAO,EAAE,eAAe,EAAE,MAAM,yCAAyC,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,wBAAwB,CAAC;AACvC,cAAc,wBAAwB,CAAC;AACvC,cAAc,sBAAsB,CAAC;AACrC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,wBAAwB,CAAC;AACvC,cAAc,sBAAsB,CAAC;AACrC,OAAO,KAAK,MAAM,MAAM,uBAAuB,CAAC;AAChD,cAAc,yBAAyB,CAAC;AACxC,cAAc,kCAAkC,CAAC;AACjD,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC;AACvC,cAAc,0BAA0B,CAAC;AACzC,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC;AAEnC,OAAO,EAAE,oBAAoB,EAAa,MAAM,qCAAqC,CAAC;AACtF,OAAO,EAAE,aAAa,EAAc,MAAM,wCAAwC,CAAC"}

package/internals/cheerio.d.ts

@@ -1,5 +1,7 @@
 import type { CheerioAPI } from 'cheerio';
 export type CheerioRoot = CheerioAPI;
+export type { CheerioAPI, Cheerio } from 'cheerio';
+export type { Element } from 'domhandler';
 /**
  * The function converts a HTML document to a plain text.
  *

package/internals/cheerio.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cheerio.d.ts","sourceRoot":"","sources":["../../src/internals/cheerio.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAK1C,MAAM,MAAM,WAAW,GAAG,UAAU,CAAC;AAOrC;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,wBAAgB,UAAU,CAAC,oBAAoB,EAAE,MAAM,GAAG,WAAW,GAAG,MAAM,CAwC7E;AAED;;;;;;;;GAQG;AACH,wBAAgB,sBAAsB,CAAC,CAAC,EAAE,UAAU,EAAE,QAAQ,SAAM,EAAE,OAAO,SAAK,GAAG,MAAM,EAAE,CAwB5F"}
+{"version":3,"file":"cheerio.d.ts","sourceRoot":"","sources":["../../src/internals/cheerio.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAK1C,MAAM,MAAM,WAAW,GAAG,UAAU,CAAC;AACrC,YAAY,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AACnD,YAAY,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAO1C;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,wBAAgB,UAAU,CAAC,oBAAoB,EAAE,MAAM,GAAG,WAAW,GAAG,MAAM,CAwC7E;AAED;;;;;;;;GAQG;AACH,wBAAgB,sBAAsB,CAAC,CAAC,EAAE,UAAU,EAAE,QAAQ,SAAM,EAAE,OAAO,SAAK,GAAG,MAAM,EAAE,CAwB5F"}

package/internals/cheerio.js.map

@@ -1 +1 @@
-{"version":3,"file":"cheerio.js","sourceRoot":"","sources":["../../src/internals/cheerio.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAInD,gIAAgI;AAChI,MAAM,eAAe,GAAG,uCAAuC,CAAC;AAChE,MAAM,gBAAgB,GAClB,sGAAsG,CAAC;AAE3G;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,MAAM,UAAU,UAAU,CAAC,oBAA0C;IACjE,IAAI,CAAC,oBAAoB;QAAE,OAAO,EAAE,CAAC;IAErC,MAAM,CAAC,GAAG,OAAO,oBAAoB,KAAK,UAAU,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACjH,IAAI,IAAI,GAAG,EAAE,CAAC;IAEd,MAAM,OAAO,GAAG,CAAC,KAAiB,EAAE,EAAE;QAClC,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACvB,qDAAqD;gBACrD,IAAI,KAAK,CAAC;gBACV,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,KAAK,KAAK;oBAAE,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC;;oBAC/D,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;gBAC5C,+EAA+E;gBAC/E,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;oBAAE,KAAK,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;gBAC9E,IAAI,IAAI,KAAK,CAAC;YAClB,CAAC;iBAAM,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;gBACvE,qCAAqC;YACzC,CAAC;iBAAM,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;gBAC/B,IAAI,IAAI,IAAI,CAAC;YACjB,CAAC;iBAAM,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;gBAC/B,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACvB,IAAI,IAAI,IAAI,CAAC;YACjB,CAAC;iBAAM,CAAC;gBACJ,2EAA2E;gBAC3E,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACvD,IAAI,UAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;oBAAE,IAAI,IAAI,IAAI,CAAC;gBACtD,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACvB,IAAI,UAAU,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;oBAAE,IAAI,IAAI,IAAI,CAAC;YACzD,CAAC;QACL,CAAC;IACL,CAAC,CAAC;IAEF,kFAAkF;IAClF,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC;IACxB,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAE7C,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC;AACvB,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,sBAAsB,CAAC,CAAa,EAAE,QAAQ,GAAG,GAAG,EAAE,OAAO,GAAG,EAAE;IAC9E,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACpC,MAAM,eAAe,GAAG,IAAI,IAAI,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAE9D,IAAI,eAAe,EAAE,CAAC;QAClB,OAAO,GAAG,eAAe,CAAC;IAC9B,CAAC;IAED,OAAO,CAAC,CAAC,QAAQ,CAAC;SACb,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;SACnC,GAAG,EAAE;SACL,MAAM,CAAC,OAAO,CAAC;SACf,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACV,yHAAyH;QACzH,MAAM,cAAc,GAAG,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,6CAA6C;QACtG,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CACX,qBAAqB,IAAI,uCAAuC;gBAC5D,2DAA2D,CAClE,CAAC;QACN,CAAC;QACD,OAAO,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC1D,CAAC,CAAC;SACD,MAAM,CAAC,OAAO,CAAa,CAAC;AACrC,CAAC"}
+{"version":3,"file":"cheerio.js","sourceRoot":"","sources":["../../src/internals/cheerio.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAMnD,gIAAgI;AAChI,MAAM,eAAe,GAAG,uCAAuC,CAAC;AAChE,MAAM,gBAAgB,GAClB,sGAAsG,CAAC;AAE3G;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,MAAM,UAAU,UAAU,CAAC,oBAA0C;IACjE,IAAI,CAAC,oBAAoB;QAAE,OAAO,EAAE,CAAC;IAErC,MAAM,CAAC,GAAG,OAAO,oBAAoB,KAAK,UAAU,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACjH,IAAI,IAAI,GAAG,EAAE,CAAC;IAEd,MAAM,OAAO,GAAG,CAAC,KAAiB,EAAE,EAAE;QAClC,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACvB,qDAAqD;gBACrD,IAAI,KAAK,CAAC;gBACV,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,KAAK,KAAK;oBAAE,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC;;oBAC/D,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;gBAC5C,+EAA+E;gBAC/E,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;oBAAE,KAAK,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;gBAC9E,IAAI,IAAI,KAAK,CAAC;YAClB,CAAC;iBAAM,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;gBACvE,qCAAqC;YACzC,CAAC;iBAAM,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;gBAC/B,IAAI,IAAI,IAAI,CAAC;YACjB,CAAC;iBAAM,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;gBAC/B,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACvB,IAAI,IAAI,IAAI,CAAC;YACjB,CAAC;iBAAM,CAAC;gBACJ,2EAA2E;gBAC3E,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACvD,IAAI,UAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;oBAAE,IAAI,IAAI,IAAI,CAAC;gBACtD,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACvB,IAAI,UAAU,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;oBAAE,IAAI,IAAI,IAAI,CAAC;YACzD,CAAC;QACL,CAAC;IACL,CAAC,CAAC;IAEF,kFAAkF;IAClF,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC;IACxB,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAE7C,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC;AACvB,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,sBAAsB,CAAC,CAAa,EAAE,QAAQ,GAAG,GAAG,EAAE,OAAO,GAAG,EAAE;IAC9E,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACpC,MAAM,eAAe,GAAG,IAAI,IAAI,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAE9D,IAAI,eAAe,EAAE,CAAC;QAClB,OAAO,GAAG,eAAe,CAAC;IAC9B,CAAC;IAED,OAAO,CAAC,CAAC,QAAQ,CAAC;SACb,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;SACnC,GAAG,EAAE;SACL,MAAM,CAAC,OAAO,CAAC;SACf,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACV,yHAAyH;QACzH,MAAM,cAAc,GAAG,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,6CAA6C;QACtG,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CACX,qBAAqB,IAAI,uCAAuC;gBAC5D,2DAA2D,CAClE,CAAC;QACN,CAAC;QACD,OAAO,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC1D,CAAC,CAAC;SACD,MAAM,CAAC,OAAO,CAAa,CAAC;AACrC,CAAC"}

package/internals/debug.d.ts

@@ -27,5 +27,6 @@ interface Request<UserData extends Dictionary = Dictionary> {
  * @param [additionalFields] Object containing additional fields to be added.
  */
 export declare function createRequestDebugInfo(request: Request, response?: IncomingMessage | Partial<BrowserResponseLike>, additionalFields?: Dictionary): Dictionary;
+export declare function getObjectType(value: unknown): string;
 export {};
 //# sourceMappingURL=debug.d.ts.map

package/internals/debug.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"debug.d.ts","sourceRoot":"","sources":["../../src/internals/debug.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAEjD,OAAO,KAAK,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAGrE,UAAU,mBAAmB;IACzB,MAAM,IAAI,MAAM,CAAC;CACpB;AAED,UAAU,OAAO,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU;IACtD,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,kBAAkB,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,QAAQ,EAAE,QAAQ,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;;;;;;;GAQG;AACH,wBAAgB,sBAAsB,CAClC,OAAO,EAAE,OAAO,EAChB,QAAQ,GAAE,eAAe,GAAG,OAAO,CAAC,mBAAmB,CAAM,EAC7D,gBAAgB,GAAE,UAAe,GAClC,UAAU,CAmBZ"}
+{"version":3,"file":"debug.d.ts","sourceRoot":"","sources":["../../src/internals/debug.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAEjD,OAAO,KAAK,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAGrE,UAAU,mBAAmB;IACzB,MAAM,IAAI,MAAM,CAAC;CACpB;AAED,UAAU,OAAO,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU;IACtD,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,kBAAkB,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,QAAQ,EAAE,QAAQ,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;;;;;;;GAQG;AACH,wBAAgB,sBAAsB,CAClC,OAAO,EAAE,OAAO,EAChB,QAAQ,GAAE,eAAe,GAAG,OAAO,CAAC,mBAAmB,CAAM,EAC7D,gBAAgB,GAAE,UAAe,GAClC,UAAU,CAmBZ;AAED,wBAAgB,aAAa,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAepD"}

package/internals/debug.js

@@ -26,4 +26,16 @@ export function createRequestDebugInfo(request, response = {}, additionalFields
         ...additionalFields,
     };
 }
+export function getObjectType(value) {
+    const simple = typeof value;
+    if (['string', 'number', 'boolean', 'bigint'].includes(simple)) {
+        return simple;
+    }
+    const objectType = Object.prototype.toString.call(value);
+    const type = objectType.match(/\[object (\w+)]/)[1];
+    if (type === 'Uint8Array') {
+        return 'Buffer';
+    }
+    return ['Date', 'Buffer', 'RegExp'].includes(type) ? type : type.toLowerCase();
+}
 //# sourceMappingURL=debug.js.map

package/internals/debug.js.map

@@ -1 +1 @@
-{"version":3,"file":"debug.js","sourceRoot":"","sources":["../../src/internals/debug.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,MAAM,IAAI,CAAC;AAqBpB;;;;;;;;GAQG;AACH,MAAM,UAAU,sBAAsB,CAClC,OAAgB,EAChB,WAA2D,EAAE,EAC7D,mBAA+B,EAAE;IAEjC,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;IACvB,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;IACxB,EAAE,CAAC,gBAAgB,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;IAEhC,OAAO;QACH,SAAS,EAAE,OAAO,CAAC,EAAE;QACrB,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,sFAAsF;QACtF,UAAU,EACN,QAAQ,IAAI,QAAQ,IAAI,QAAQ,CAAC,MAAM,YAAY,QAAQ;YACvD,CAAC,CAAC,QAAQ,CAAC,MAAM,EAAE;YACnB,CAAC,CAAE,QAA4B,CAAC,UAAU;QAClD,GAAG,gBAAgB;KACtB,CAAC;AACN,CAAC"}
+{"version":3,"file":"debug.js","sourceRoot":"","sources":["../../src/internals/debug.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,MAAM,IAAI,CAAC;AAqBpB;;;;;;;;GAQG;AACH,MAAM,UAAU,sBAAsB,CAClC,OAAgB,EAChB,WAA2D,EAAE,EAC7D,mBAA+B,EAAE;IAEjC,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;IACvB,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;IACxB,EAAE,CAAC,gBAAgB,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;IAEhC,OAAO;QACH,SAAS,EAAE,OAAO,CAAC,EAAE;QACrB,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,sFAAsF;QACtF,UAAU,EACN,QAAQ,IAAI,QAAQ,IAAI,QAAQ,CAAC,MAAM,YAAY,QAAQ;YACvD,CAAC,CAAC,QAAQ,CAAC,MAAM,EAAE;YACnB,CAAC,CAAE,QAA4B,CAAC,UAAU;QAClD,GAAG,gBAAgB;KACtB,CAAC;AACN,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,KAAc;IACxC,MAAM,MAAM,GAAG,OAAO,KAAK,CAAC;IAE5B,IAAI,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAC7D,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACzD,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,iBAAiB,CAAE,CAAC,CAAC,CAAC,CAAC;IAErD,IAAI,IAAI,KAAK,YAAY,EAAE,CAAC;QACxB,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;AACnF,CAAC"}

package/internals/extract-urls.d.ts

@@ -1,3 +1,4 @@
+import type { BaseHttpClient } from '@crawlee/types';
 export interface DownloadListOfUrlsOptions {
     /**
      * URL to the file
@@ -16,6 +17,10 @@ export interface DownloadListOfUrlsOptions {
     urlRegExp?: RegExp;
     /** Allows to use a proxy for the download request. */
     proxyUrl?: string;
+    /**
+     * Custom HTTP client to use for downloading the file.
+     */
+    httpClient?: BaseHttpClient;
 }
 /**
  * Returns a promise that resolves to an array of urls parsed from the resource available at the provided url.

package/internals/extract-urls.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"extract-urls.d.ts","sourceRoot":"","sources":["../../src/internals/extract-urls.ts"],"names":[],"mappings":"AAKA,MAAM,WAAW,yBAAyB;IACtC;;OAEG;IACH,GAAG,EAAE,MAAM,CAAC;IAEZ;;;OAGG;IACH,QAAQ,CAAC,EAAE,cAAc,CAAC;IAE1B;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,sDAAsD;IACtD,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;GAGG;AACH,wBAAsB,kBAAkB,CAAC,OAAO,EAAE,yBAAyB,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAuB9F;AAED,MAAM,WAAW,kBAAkB;IAC/B;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,kBAAkB,GAAG,MAAM,EAAE,CAiBjE;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAMhF"}
+{"version":3,"file":"extract-urls.d.ts","sourceRoot":"","sources":["../../src/internals/extract-urls.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAKrD,MAAM,WAAW,yBAAyB;IACtC;;OAEG;IACH,GAAG,EAAE,MAAM,CAAC;IAEZ;;;OAGG;IACH,QAAQ,CAAC,EAAE,cAAc,CAAC;IAE1B;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,sDAAsD;IACtD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;OAEG;IACH,UAAU,CAAC,EAAE,cAAc,CAAC;CAC/B;AAED;;;GAGG;AACH,wBAAsB,kBAAkB,CAAC,OAAO,EAAE,yBAAyB,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAkC9F;AAED,MAAM,WAAW,kBAAkB;IAC/B;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,kBAAkB,GAAG,MAAM,EAAE,CAiBjE;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAMhF"}

package/internals/extract-urls.js

@@ -1,4 +1,4 @@
-import { gotScraping } from 'got-scraping';
+import { ImpitHttpClient } from '@crawlee/impit-client';
 import ow from 'ow';
 import { URL_NO_COMMAS_REGEX } from './general.js';
 /**
@@ -11,15 +11,19 @@ export async function downloadListOfUrls(options) {
         encoding: ow.optional.string,
         urlRegExp: ow.optional.regExp,
         proxyUrl: ow.optional.string,
+        httpClient: ow.optional.object,
     }));
-    const { url, encoding = 'utf8', urlRegExp = URL_NO_COMMAS_REGEX, proxyUrl } = options;
+    const { url, encoding = 'utf8', urlRegExp = URL_NO_COMMAS_REGEX, proxyUrl, httpClient = new ImpitHttpClient(), } = options;
     // Try to detect wrong urls and fix them. Currently, detects only sharing url instead of csv download one.
     const match = url.match(/^(https:\/\/docs\.google\.com\/spreadsheets\/d\/(?:\w|-)+)\/?/);
     let fixedUrl = url;
     if (match) {
         fixedUrl = `${match[1]}/gviz/tq?tqx=out:csv`;
     }
-    const { body: string } = await gotScraping({ url: fixedUrl, encoding, proxyUrl });
+    const response = await httpClient.sendRequest(new Request(fixedUrl, { method: 'GET' }), {
+        proxyUrl,
+    });
+    const string = new TextDecoder(encoding).decode(new Uint8Array(await response.arrayBuffer()));
     return extractUrls({ string, urlRegExp });
 }
 /**

package/internals/extract-urls.js.map

@@ -1 +1 @@
-{"version":3,"file":"extract-urls.js","sourceRoot":"","sources":["../../src/internals/extract-urls.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAyBnD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,OAAkC;IACvE,EAAE,CACE,OAAc,EACd,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QACjB,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG;QAClB,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC7B,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;KAC/B,CAAC,CACL,CAAC;IACF,MAAM,EAAE,GAAG,EAAE,QAAQ,GAAG,MAAM,EAAE,SAAS,GAAG,mBAAmB,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;IAEtF,0GAA0G;IAC1G,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,+DAA+D,CAAC,CAAC;IACzF,IAAI,QAAQ,GAAG,GAAG,CAAC;IAEnB,IAAI,KAAK,EAAE,CAAC;QACR,QAAQ,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,sBAAsB,CAAC;IACjD,CAAC;IAED,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,WAAW,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC;IAElF,OAAO,WAAW,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;AAC9C,CAAC;AAeD;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,OAA2B;IACnD,EAAE,CACE,OAAc,EACd,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QACjB,MAAM,EAAE,EAAE,CAAC,MAAM;QACjB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;KAChC,CAAC,CACL,CAAC;IACF,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACzC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,mBAAmB,CAAC;IAE3D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAClD,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY,EAAE,OAAe;IACxD,IAAI,CAAC;QACD,OAAO,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;IACvC,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,SAAS,CAAC;IACrB,CAAC;AACL,CAAC"}
+{"version":3,"file":"extract-urls.js","sourceRoot":"","sources":["../../src/internals/extract-urls.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AA8BnD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,OAAkC;IACvE,EAAE,CACE,OAAc,EACd,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QACjB,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG;QAClB,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC7B,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;KACjC,CAAC,CACL,CAAC;IACF,MAAM,EACF,GAAG,EACH,QAAQ,GAAG,MAAM,EACjB,SAAS,GAAG,mBAAmB,EAC/B,QAAQ,EACR,UAAU,GAAG,IAAI,eAAe,EAAE,GACrC,GAAG,OAAO,CAAC;IAEZ,0GAA0G;IAC1G,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,+DAA+D,CAAC,CAAC;IACzF,IAAI,QAAQ,GAAG,GAAG,CAAC;IAEnB,IAAI,KAAK,EAAE,CAAC;QACR,QAAQ,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,sBAAsB,CAAC;IACjD,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,WAAW,CAAC,IAAI,OAAO,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE;QACpF,QAAQ;KACX,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,UAAU,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;IAE9F,OAAO,WAAW,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;AAC9C,CAAC;AAeD;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,OAA2B;IACnD,EAAE,CACE,OAAc,EACd,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QACjB,MAAM,EAAE,EAAE,CAAC,MAAM;QACjB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;KAChC,CAAC,CACL,CAAC;IACF,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACzC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,mBAAmB,CAAC;IAE3D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAClD,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY,EAAE,OAAe;IACxD,IAAI,CAAC;QACD,OAAO,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;IACvC,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,SAAS,CAAC;IACrB,CAAC;AACL,CAAC"}

package/internals/iterables.d.ts

@@ -0,0 +1,108 @@
+/**
+ * Type guard that checks if a value is iterable (has Symbol.iterator).
+ * @internal
+ *
+ * **Example usage:**
+ * ```ts
+ * if (isIterable(someValue)) {
+ *   for (const item of someValue) {
+ *     console.log(item);
+ *   }
+ * }
+ * ```
+ */
+export declare function isIterable<T>(value: unknown): value is Iterable<T>;
+/**
+ * Type guard that checks if a value is async iterable (has Symbol.asyncIterator).
+ * @internal
+ *
+ * **Example usage:**
+ * ```ts
+ * if (isAsyncIterable(someValue)) {
+ *   for await (const item of someValue) {
+ *     console.log(item);
+ *   }
+ * }
+ * ```
+ */
+export declare function isAsyncIterable<T>(value: unknown): value is AsyncIterable<T>;
+/**
+ * Converts any iterable or async iterable to an async iterable.
+ * @internal
+ *
+ * @yields Each item from the input iterable
+ *
+ * **Example usage:**
+ * ```ts
+ * const syncArray = [1, 2, 3];
+ * for await (const item of asyncifyIterable(syncArray)) {
+ *   console.log(item); // 1, 2, 3
+ * }
+ * ```
+ */
+export declare function asyncifyIterable<T>(iterable: Iterable<T> | AsyncIterable<T>): AsyncIterable<T>;
+/**
+ * Lazily splits the input async iterable into chunks of specified size.
+ * The last chunk may contain fewer items if the total number of items
+ * is not evenly divisible by the chunk size.
+ * @internal
+ *
+ * @yields Arrays of items, each containing up to chunkSize items
+ *
+ * **Example usage:**
+ * ```ts
+ * const numbers = async function* () {
+ *   for (let i = 1; i <= 10; i++) yield i;
+ * };
+ *
+ * for await (const chunk of chunkedAsyncIterable(numbers(), 3)) {
+ *   console.log(chunk); // [1, 2, 3], [4, 5, 6], [7, 8, 9], [10]
+ * }
+ * ```
+ */
+export declare function chunkedAsyncIterable<T>(iterable: AsyncIterable<T> | Iterable<T>, chunkSize: number): AsyncIterable<T[]>;
+/**
+ * An async iterator that also supports peeking at the next value without consuming it.
+ * Extends both AsyncIterator and AsyncIterable interfaces.
+ * @internal
+ */
+export interface PeekableAsyncIterator<T> extends AsyncIterator<T>, AsyncIterable<T> {
+    /**
+     * Peeks at the next value without consuming it from the iterator.
+     * Subsequent calls to peek() will return the same value until next() is called.
+     *
+     * @returns Promise that resolves to the next value, or undefined if the iterator is exhausted
+     */
+    peek(): Promise<T | undefined>;
+}
+/**
+ * An async iterable that yields peekable async iterators.
+ * @internal
+ */
+export interface PeekableAsyncIterable<T> extends AsyncIterable<T> {
+    [Symbol.asyncIterator](): PeekableAsyncIterator<T>;
+}
+/**
+ * Wraps an async iterable to provide peek functionality, allowing you to look at
+ * the next value without consuming it from the iterator.
+ * @internal
+ *
+ * @param iterable - The async iterable to make peekable
+ *
+ * **Example usage:**
+ * ```ts
+ * const numbers = async function* () {
+ *   yield 1; yield 2; yield 3;
+ * };
+ *
+ * const peekable = peekableAsyncIterable(numbers());
+ * const iterator = peekable[Symbol.asyncIterator]();
+ *
+ * console.log(await iterator.peek()); // 1 (doesn't consume)
+ * console.log(await iterator.peek()); // 1 (still doesn't consume)
+ * console.log(await iterator.next()); // { value: 1, done: false } (now consumed)
+ * console.log(await iterator.peek()); // 2 (next value)
+ * ```
+ */
+export declare function peekableAsyncIterable<T>(iterable: AsyncIterable<T> | Iterable<T>): PeekableAsyncIterable<T>;
+//# sourceMappingURL=iterables.d.ts.map

package/internals/iterables.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"iterables.d.ts","sourceRoot":"","sources":["../../src/internals/iterables.ts"],"names":[],"mappings":"AAEA;;;;;;;;;;;;GAYG;AACH,wBAAgB,UAAU,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,QAAQ,CAAC,CAAC,CAAC,CAUlE;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,eAAe,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,aAAa,CAAC,CAAC,CAAC,CAM5E;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAuB,gBAAgB,CAAC,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,CAErG;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAuB,oBAAoB,CAAC,CAAC,EACzC,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,EACxC,SAAS,EAAE,MAAM,GAClB,aAAa,CAAC,CAAC,EAAE,CAAC,CAmBpB;AAED;;;;GAIG;AACH,MAAM,WAAW,qBAAqB,CAAC,CAAC,CAAE,SAAQ,aAAa,CAAC,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC,CAAC;IAChF;;;;;OAKG;IACH,IAAI,IAAI,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC;CAClC;AAED;;;GAGG;AACH,MAAM,WAAW,qBAAqB,CAAC,CAAC,CAAE,SAAQ,aAAa,CAAC,CAAC,CAAC;IAC9D,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,qBAAqB,CAAC,CAAC,CAAC,CAAC;CACtD;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,qBAAqB,CAAC,CAAC,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,qBAAqB,CAAC,CAAC,CAAC,CA+D3G"}

package/internals/iterables.js

@@ -0,0 +1,167 @@
+import { inspect } from 'node:util';
+/**
+ * Type guard that checks if a value is iterable (has Symbol.iterator).
+ * @internal
+ *
+ * **Example usage:**
+ * ```ts
+ * if (isIterable(someValue)) {
+ *   for (const item of someValue) {
+ *     console.log(item);
+ *   }
+ * }
+ * ```
+ */
+export function isIterable(value) {
+    if (value == null || typeof value === 'string' || ArrayBuffer.isView(value)) {
+        return false;
+    }
+    if (Array.isArray(value)) {
+        return true;
+    }
+    return typeof Object(value)[Symbol.iterator] === 'function';
+}
+/**
+ * Type guard that checks if a value is async iterable (has Symbol.asyncIterator).
+ * @internal
+ *
+ * **Example usage:**
+ * ```ts
+ * if (isAsyncIterable(someValue)) {
+ *   for await (const item of someValue) {
+ *     console.log(item);
+ *   }
+ * }
+ * ```
+ */
+export function isAsyncIterable(value) {
+    if (value == null || typeof value === 'string' || ArrayBuffer.isView(value)) {
+        return false;
+    }
+    return typeof Object(value)[Symbol.asyncIterator] === 'function';
+}
+/**
+ * Converts any iterable or async iterable to an async iterable.
+ * @internal
+ *
+ * @yields Each item from the input iterable
+ *
+ * **Example usage:**
+ * ```ts
+ * const syncArray = [1, 2, 3];
+ * for await (const item of asyncifyIterable(syncArray)) {
+ *   console.log(item); // 1, 2, 3
+ * }
+ * ```
+ */
+export async function* asyncifyIterable(iterable) {
+    yield* iterable;
+}
+/**
+ * Lazily splits the input async iterable into chunks of specified size.
+ * The last chunk may contain fewer items if the total number of items
+ * is not evenly divisible by the chunk size.
+ * @internal
+ *
+ * @yields Arrays of items, each containing up to chunkSize items
+ *
+ * **Example usage:**
+ * ```ts
+ * const numbers = async function* () {
+ *   for (let i = 1; i <= 10; i++) yield i;
+ * };
+ *
+ * for await (const chunk of chunkedAsyncIterable(numbers(), 3)) {
+ *   console.log(chunk); // [1, 2, 3], [4, 5, 6], [7, 8, 9], [10]
+ * }
+ * ```
+ */
+export async function* chunkedAsyncIterable(iterable, chunkSize) {
+    if (typeof chunkSize !== 'number' || chunkSize < 1) {
+        throw new Error(`Chunk size must be a positive number (${inspect(chunkSize)}) received`);
+    }
+    let chunk = [];
+    for await (const item of iterable) {
+        chunk.push(item);
+        if (chunk.length >= chunkSize) {
+            yield chunk;
+            chunk = [];
+        }
+    }
+    if (chunk.length) {
+        yield chunk;
+    }
+}
+/**
+ * Wraps an async iterable to provide peek functionality, allowing you to look at
+ * the next value without consuming it from the iterator.
+ * @internal
+ *
+ * @param iterable - The async iterable to make peekable
+ *
+ * **Example usage:**
+ * ```ts
+ * const numbers = async function* () {
+ *   yield 1; yield 2; yield 3;
+ * };
+ *
+ * const peekable = peekableAsyncIterable(numbers());
+ * const iterator = peekable[Symbol.asyncIterator]();
+ *
+ * console.log(await iterator.peek()); // 1 (doesn't consume)
+ * console.log(await iterator.peek()); // 1 (still doesn't consume)
+ * console.log(await iterator.next()); // { value: 1, done: false } (now consumed)
+ * console.log(await iterator.peek()); // 2 (next value)
+ * ```
+ */
+export function peekableAsyncIterable(iterable) {
+    const iterator = asyncifyIterable(iterable)[Symbol.asyncIterator]();
+    let peekedValue;
+    let isExhausted = false;
+    const peekableIterator = {
+        async next() {
+            // If we have peeked a value, return it and clear the peek
+            if (peekedValue !== undefined) {
+                const result = peekedValue;
+                peekedValue = undefined;
+                if (result.done) {
+                    isExhausted = true;
+                    return { done: true, value: undefined };
+                }
+                return { done: false, value: result.value };
+            }
+            if (isExhausted) {
+                return { done: true, value: undefined };
+            }
+            const result = await iterator.next();
+            if (result.done) {
+                isExhausted = true;
+            }
+            return result;
+        },
+        async peek() {
+            if (peekedValue !== undefined) {
+                return peekedValue.done ? undefined : peekedValue.value;
+            }
+            if (isExhausted) {
+                return undefined;
+            }
+            const result = await iterator.next();
+            peekedValue = { done: result.done ?? false, value: result.value };
+            if (result.done) {
+                isExhausted = true;
+                return undefined;
+            }
+            return result.value;
+        },
+        [Symbol.asyncIterator]() {
+            return this;
+        },
+    };
+    return {
+        [Symbol.asyncIterator]() {
+            return peekableIterator;
+        },
+    };
+}
+//# sourceMappingURL=iterables.js.map

package/internals/iterables.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"iterables.js","sourceRoot":"","sources":["../../src/internals/iterables.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,UAAU,CAAI,KAAc;IACxC,IAAI,KAAK,IAAI,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1E,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,OAAO,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,UAAU,CAAC;AAChE,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,eAAe,CAAI,KAAc;IAC7C,IAAI,KAAK,IAAI,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1E,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,OAAO,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,UAAU,CAAC;AACrE,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,gBAAgB,CAAI,QAAwC;IAC/E,KAAK,CAAC,CAAC,QAAQ,CAAC;AACpB,CAAC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,oBAAoB,CACvC,QAAwC,EACxC,SAAiB;IAEjB,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QACjD,MAAM,IAAI,KAAK,CAAC,yCAAyC,OAAO,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;IAC7F,CAAC;IAED,IAAI,KAAK,GAAQ,EAAE,CAAC;IAEpB,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEjB,IAAI,KAAK,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;YAC5B,MAAM,KAAK,CAAC;YACZ,KAAK,GAAG,EAAE,CAAC;QACf,CAAC;IACL,CAAC;IAED,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACf,MAAM,KAAK,CAAC;IAChB,CAAC;AACL,CAAC;AAyBD;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,UAAU,qBAAqB,CAAI,QAAwC;IAC7E,MAAM,QAAQ,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;IACpE,IAAI,WAAoD,CAAC;IACzD,IAAI,WAAW,GAAG,KAAK,CAAC;IAExB,MAAM,gBAAgB,GAA6B;QAC/C,KAAK,CAAC,IAAI;YACN,0DAA0D;YAC1D,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;gBAC5B,MAAM,MAAM,GAAG,WAAW,CAAC;gBAC3B,WAAW,GAAG,SAAS,CAAC;gBAExB,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;oBACd,WAAW,GAAG,IAAI,CAAC;oBACnB,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;gBAC5C,CAAC;gBAED,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC;YAChD,CAAC;YAED,IAAI,WAAW,EAAE,CAAC;gBACd,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;YAC5C,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAErC,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBACd,WAAW,GAAG,IAAI,CAAC;YACvB,CAAC;YAED,OAAO,MAAM,CAAC;QAClB,CAAC;QAED,KAAK,CAAC,IAAI;YACN,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;gBAC5B,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC;YAC5D,CAAC;YAED,IAAI,WAAW,EAAE,CAAC;gBACd,OAAO,SAAS,CAAC;YACrB,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACrC,WAAW,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC;YAElE,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBACd,WAAW,GAAG,IAAI,CAAC;gBACnB,OAAO,SAAS,CAAC;YACrB,CAAC;YAED,OAAO,MAAM,CAAC,KAAK,CAAC;QACxB,CAAC;QAED,CAAC,MAAM,CAAC,aAAa,CAAC;YAClB,OAAO,IAAI,CAAC;QAChB,CAAC;KACJ,CAAC;IAEF,OAAO;QACH,CAAC,MAAM,CAAC,aAAa,CAAC;YAClB,OAAO,gBAAgB,CAAC;QAC5B,CAAC;KACJ,CAAC;AACN,CAAC"}

package/internals/robots.d.ts

@@ -1,3 +1,4 @@
+import type { BaseHttpClient } from '@crawlee/types';
 import { Sitemap } from './sitemap.js';
 /**
  * Loads and queries information from a [robots.txt file](https://en.wikipedia.org/wiki/Robots.txt).
@@ -26,7 +27,10 @@ export declare class RobotsTxtFile {
      * @param url the URL to fetch robots.txt for
      * @param [proxyUrl] a proxy to be used for fetching the robots.txt file
      */
-    static find(url: string, proxyUrl?: string): Promise<RobotsTxtFile>;
+    static find(url: string, options?: {
+        proxyUrl?: string;
+        httpClient?: BaseHttpClient;
+    }): Promise<RobotsTxtFile>;
     /**
      * Allows providing the URL and robots.txt content explicitly instead of loading it from the target site.
      * @param url the URL for robots.txt file
@@ -34,7 +38,10 @@ export declare class RobotsTxtFile {
      * @param [proxyUrl] a proxy to be used for fetching the robots.txt file
      */
     static from(url: string, content: string, proxyUrl?: string): RobotsTxtFile;
-    protected static load(url: string, proxyUrl?: string): Promise<RobotsTxtFile>;
+    protected static load(url: string, options?: {
+        proxyUrl?: string;
+        httpClient?: BaseHttpClient;
+    }): Promise<RobotsTxtFile>;
     /**
      * Check if a URL should be crawled by robots.
      * @param url the URL to check against the rules in robots.txt

package/internals/robots.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"robots.d.ts","sourceRoot":"","sources":["../../src/internals/robots.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAIvC;;;;;;;;;;;;;;;;;GAiBG;AACH,qBAAa,aAAa;IAElB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,QAAQ,CAAC;IAFrB,OAAO;IAKP;;;;OAIG;WACU,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC;IAQzE;;;;;OAKG;IACH,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,aAAa;qBAKpD,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC;IAiCnF;;;;OAIG;IACH,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,SAAM,GAAG,OAAO;IAIhD;;OAEG;IACH,WAAW,IAAI,MAAM,EAAE;IAIvB;;OAEG;IACG,aAAa,IAAI,OAAO,CAAC,OAAO,CAAC;IAIvC;;OAEG;IACG,qBAAqB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;CAGnD;AAGD,OAAO,EAAE,aAAa,IAAI,UAAU,EAAE,CAAC"}
+{"version":3,"file":"robots.d.ts","sourceRoot":"","sources":["../../src/internals/robots.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAIrD,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC;;;;;;;;;;;;;;;;;GAiBG;AACH,qBAAa,aAAa;IAElB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,QAAQ,CAAC;IAFrB,OAAO;IAKP;;;;OAIG;WACU,IAAI,CACb,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,cAAc,CAAA;KAAE,GAC7D,OAAO,CAAC,aAAa,CAAC;IAQzB;;;;;OAKG;IACH,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,aAAa;qBAKpD,IAAI,CACvB,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,cAAc,CAAA;KAAE,GAC7D,OAAO,CAAC,aAAa,CAAC;IA6BzB;;;;OAIG;IACH,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,SAAM,GAAG,OAAO;IAIhD;;OAEG;IACH,WAAW,IAAI,MAAM,EAAE;IAIvB;;OAEG;IACG,aAAa,IAAI,OAAO,CAAC,OAAO,CAAC;IAIvC;;OAEG;IACG,qBAAqB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;CAGnD;AAGD,OAAO,EAAE,aAAa,IAAI,UAAU,EAAE,CAAC"}

package/internals/robots.js

@@ -1,7 +1,6 @@
-import { gotScraping } from 'got-scraping';
+import { ImpitHttpClient } from '@crawlee/impit-client';
 import robotsParser from 'robots-parser';
 import { Sitemap } from './sitemap.js';
-let HTTPError;
 /**
  * Loads and queries information from a [robots.txt file](https://en.wikipedia.org/wiki/Robots.txt).
  *
@@ -32,11 +31,11 @@ export class RobotsTxtFile {
      * @param url the URL to fetch robots.txt for
      * @param [proxyUrl] a proxy to be used for fetching the robots.txt file
      */
-    static async find(url, proxyUrl) {
+    static async find(url, options) {
         const robotsTxtFileUrl = new URL(url);
         robotsTxtFileUrl.pathname = '/robots.txt';
         robotsTxtFileUrl.search = '';
-        return RobotsTxtFile.load(robotsTxtFileUrl.toString(), proxyUrl);
+        return RobotsTxtFile.load(robotsTxtFileUrl.toString(), options);
     }
     /**
      * Allows providing the URL and robots.txt content explicitly instead of loading it from the target site.
@@ -48,33 +47,26 @@ export class RobotsTxtFile {
         // @ts-ignore
         return new RobotsTxtFile(robotsParser(url, content), proxyUrl);
     }
-    static async load(url, proxyUrl) {
-        if (!HTTPError) {
-            HTTPError = (await import('got-scraping')).HTTPError;
+    static async load(url, options) {
+        const { proxyUrl, httpClient = new ImpitHttpClient({ followRedirects: true }) } = options || {};
+        const response = await httpClient.sendRequest(new Request(url, { method: 'GET' }), {
+            proxyUrl,
+        });
+        if (response.status < 200 || response.status >= 300) {
+            throw new Error(`Failed to load robots.txt from ${url}: HTTP ${response.status}`);
         }
-        try {
-            const response = await gotScraping({
-                url,
-                proxyUrl,
-                method: 'GET',
-                responseType: 'text',
-            });
-            // @ts-ignore
-            return new RobotsTxtFile(robotsParser(url.toString(), response.body), proxyUrl);
-        }
-        catch (e) {
-            if (e instanceof HTTPError && e.response.statusCode === 404) {
-                return new RobotsTxtFile({
-                    isAllowed() {
-                        return true;
-                    },
-                    getSitemaps() {
-                        return [];
-                    },
-                }, proxyUrl);
-            }
-            throw e;
+        if (response.status === 404) {
+            return new RobotsTxtFile({
+                isAllowed() {
+                    return true;
+                },
+                getSitemaps() {
+                    return [];
+                },
+            }, proxyUrl);
         }
+        // @ts-ignore
+        return new RobotsTxtFile(robotsParser(url.toString(), await response.text()), proxyUrl);
     }
     /**
      * Check if a URL should be crawled by robots.

package/internals/robots.js.map

@@ -1 +1 @@
-{"version":3,"file":"robots.js","sourceRoot":"","sources":["../../src/internals/robots.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAE3C,OAAO,YAAY,MAAM,eAAe,CAAC;AAEzC,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC,IAAI,SAAgC,CAAC;AAErC;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,OAAO,aAAa;IAEV;IACA;IAFZ,YACY,MAAgD,EAChD,QAAiB;QADjB,WAAM,GAAN,MAAM,CAA0C;QAChD,aAAQ,GAAR,QAAQ,CAAS;IAC1B,CAAC;IAEJ;;;;OAIG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAW,EAAE,QAAiB;QAC5C,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACtC,gBAAgB,CAAC,QAAQ,GAAG,aAAa,CAAC;QAC1C,gBAAgB,CAAC,MAAM,GAAG,EAAE,CAAC;QAE7B,OAAO,aAAa,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,EAAE,QAAQ,CAAC,CAAC;IACrE,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,IAAI,CAAC,GAAW,EAAE,OAAe,EAAE,QAAiB;QACvD,aAAa;QACb,OAAO,IAAI,aAAa,CAAC,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC;IACnE,CAAC;IAES,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAW,EAAE,QAAiB;QACtD,IAAI,CAAC,SAAS,EAAE,CAAC;YACb,SAAS,GAAG,CAAC,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC;QACzD,CAAC;QAED,IAAI,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC;gBAC/B,GAAG;gBACH,QAAQ;gBACR,MAAM,EAAE,KAAK;gBACb,YAAY,EAAE,MAAM;aACvB,CAAC,CAAC;YAEH,aAAa;YACb,OAAO,IAAI,aAAa,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,QAAQ,CAAC,CAAC;QACpF,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,YAAY,SAAS,IAAI,CAAC,CAAC,QAAQ,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;gBAC1D,OAAO,IAAI,aAAa,CACpB;oBACI,SAAS;wBACL,OAAO,IAAI,CAAC;oBAChB,CAAC;oBACD,WAAW;wBACP,OAAO,EAAE,CAAC;oBACd,CAAC;iBACJ,EACD,QAAQ,CACX,CAAC;YACN,CAAC;YACD,MAAM,CAAC,CAAC;QACZ,CAAC;IACL,CAAC;IAED;;;;OAIG;IACH,SAAS,CAAC,GAAW,EAAE,SAAS,GAAG,GAAG;QAClC,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,EAAE,SAAS,CAAC,IAAI,IAAI,CAAC,CAAC,+FAA+F;IACzJ,CAAC;IAED;;OAEG;IACH,WAAW;QACP,OAAO,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa;QACf,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;IAClE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,qBAAqB;QACvB,OAAO,CAAC,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC,IAAI,CAAC;IAC7C,CAAC;CACJ;AAED,+BAA+B;AAC/B,OAAO,EAAE,aAAa,IAAI,UAAU,EAAE,CAAC"}
+{"version":3,"file":"robots.js","sourceRoot":"","sources":["../../src/internals/robots.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAGxD,OAAO,YAAY,MAAM,eAAe,CAAC;AAEzC,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,OAAO,aAAa;IAEV;IACA;IAFZ,YACY,MAAgD,EAChD,QAAiB;QADjB,WAAM,GAAN,MAAM,CAA0C;QAChD,aAAQ,GAAR,QAAQ,CAAS;IAC1B,CAAC;IAEJ;;;;OAIG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CACb,GAAW,EACX,OAA4D;QAE5D,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACtC,gBAAgB,CAAC,QAAQ,GAAG,aAAa,CAAC;QAC1C,gBAAgB,CAAC,MAAM,GAAG,EAAE,CAAC;QAE7B,OAAO,aAAa,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,EAAE,OAAO,CAAC,CAAC;IACpE,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,IAAI,CAAC,GAAW,EAAE,OAAe,EAAE,QAAiB;QACvD,aAAa;QACb,OAAO,IAAI,aAAa,CAAC,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC;IACnE,CAAC;IAES,MAAM,CAAC,KAAK,CAAC,IAAI,CACvB,GAAW,EACX,OAA4D;QAE5D,MAAM,EAAE,QAAQ,EAAE,UAAU,GAAG,IAAI,eAAe,CAAC,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,OAAO,IAAI,EAAE,CAAC;QAEhG,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,WAAW,CAAC,IAAI,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE;YAC/E,QAAQ;SACX,CAAC,CAAC;QAEH,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;YAClD,MAAM,IAAI,KAAK,CAAC,kCAAkC,GAAG,UAAU,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QACtF,CAAC;QAED,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YAC1B,OAAO,IAAI,aAAa,CACpB;gBACI,SAAS;oBACL,OAAO,IAAI,CAAC;gBAChB,CAAC;gBACD,WAAW;oBACP,OAAO,EAAE,CAAC;gBACd,CAAC;aACJ,EACD,QAAQ,CACX,CAAC;QACN,CAAC;QAED,aAAa;QACb,OAAO,IAAI,aAAa,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC;IAC5F,CAAC;IAED;;;;OAIG;IACH,SAAS,CAAC,GAAW,EAAE,SAAS,GAAG,GAAG;QAClC,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,EAAE,SAAS,CAAC,IAAI,IAAI,CAAC,CAAC,+FAA+F;IACzJ,CAAC;IAED;;OAEG;IACH,WAAW;QACP,OAAO,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa;QACf,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;IAClE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,qBAAqB;QACvB,OAAO,CAAC,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC,IAAI,CAAC;IAC7C,CAAC;CACJ;AAED,+BAA+B;AAC/B,OAAO,EAAE,aAAa,IAAI,UAAU,EAAE,CAAC"}