@crawlee/core 4.0.0-beta.2 → 4.0.0-beta.21

package/enqueue_links/shared.js

@@ -1,5 +1,5 @@
 import { URL } from 'node:url';
-import { minimatch } from 'minimatch';
+import { Minimatch } from 'minimatch';
 import { purlToRegExp } from '@apify/pseudo_url';
 import { Request } from '../request.js';
 export { tryAbsoluteURL } from '@crawlee/utils';
@@ -112,22 +112,25 @@ export function constructRegExpObjectsFromRegExps(regexps) {
 /**
  * @ignore
  */
-export function createRequests(requestOptions, urlPatternObjects, excludePatternObjects = [], strategy) {
+export function createRequests(requestOptions, urlPatternObjects, excludePatternObjects = [], strategy, onSkippedUrl) {
+    const excludePatternObjectMatchers = excludePatternObjects.map(createPatternObjectMatcher);
+    const urlPatternObjectMatchers = urlPatternObjects?.map(createPatternObjectMatcher);
     return requestOptions
         .map((opts) => ({ url: typeof opts === 'string' ? opts : opts.url, opts }))
         .filter(({ url }) => {
-        return !excludePatternObjects.some((excludePatternObject) => {
-            const { regexp, glob } = excludePatternObject;
-            return (regexp && url.match(regexp)) || (glob && minimatch(url, glob, { nocase: true }));
-        });
+        const matchesExcludePatterns = excludePatternObjectMatchers.some(({ match }) => match(url));
+        if (matchesExcludePatterns) {
+            onSkippedUrl?.(url);
+        }
+        return !matchesExcludePatterns;
     })
         .map(({ url, opts }) => {
-        if (!urlPatternObjects || !urlPatternObjects.length) {
+        if (!urlPatternObjectMatchers || !urlPatternObjectMatchers.length) {
             return new Request(typeof opts === 'string' ? { url: opts, enqueueStrategy: strategy } : { ...opts });
         }
-        for (const urlPatternObject of urlPatternObjects) {
-            const { regexp, glob, ...requestRegExpOptions } = urlPatternObject;
-            if ((regexp && url.match(regexp)) || (glob && minimatch(url, glob, { nocase: true }))) {
+        for (const urlPatternObject of urlPatternObjectMatchers) {
+            const { match, glob, regexp, ...requestRegExpOptions } = urlPatternObject;
+            if (match(url)) {
                 const request = typeof opts === 'string'
                     ? { url: opts, ...requestRegExpOptions, enqueueStrategy: strategy }
                     : { ...opts, ...requestRegExpOptions, enqueueStrategy: strategy };
@@ -135,23 +138,24 @@ export function createRequests(requestOptions, urlPatternObjects, excludePattern
             }
         }
         // didn't match any positive pattern
+        onSkippedUrl?.(url);
         return null;
     })
         .filter((request) => request);
 }
-export function filterRequestsByPatterns(requests, patterns) {
+export function filterRequestsByPatterns(requests, patterns, onSkippedUrl) {
     if (!patterns?.length) {
         return requests;
     }
     const filtered = [];
+    const patternMatchers = patterns?.map(createPatternObjectMatcher);
     for (const request of requests) {
-        for (const urlPatternObject of patterns) {
-            const { regexp, glob } = urlPatternObject;
-            if ((regexp && request.url.match(regexp)) || (glob && minimatch(request.url, glob, { nocase: true }))) {
-                filtered.push(request);
-                // Break the pattern loop, as we already matched this request once
-                break;
-            }
+        const matchingPattern = patternMatchers.find(({ match }) => match(request.url));
+        if (matchingPattern !== undefined) {
+            filtered.push(request);
+        }
+        else {
+            onSkippedUrl?.(request.url);
         }
     }
     return filtered;
@@ -187,4 +191,22 @@ export function createRequestOptions(sources, options = {}) {
         return requestOptions;
     });
 }
+/**
+ * @ignore
+ */
+function createPatternObjectMatcher(urlPatternObject) {
+    const { regexp, glob } = urlPatternObject;
+    let match;
+    if (regexp) {
+        match = (url) => regexp.test(url);
+    }
+    else if (glob) {
+        const m = new Minimatch(glob, { nocase: true });
+        match = (url) => m.match(url);
+    }
+    else {
+        match = () => false;
+    }
+    return { ...urlPatternObject, match };
+}
 //# sourceMappingURL=shared.js.map

package/enqueue_links/shared.js.map

@@ -1 +1 @@
-{"version":3,"file":"shared.js","sourceRoot":"","sources":["../../src/enqueue_links/shared.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAE/B,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEtC,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAGjD,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAGxC,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAEhD,MAAM,4BAA4B,GAAG,IAAI,CAAC;AAE1C;;;;;GAKG;AACH,MAAM,wBAAwB,GAAG,IAAI,GAAG,EAAE,CAAC;AA4B3C;;GAEG;AACH,MAAM,UAAU,8BAA8B,CAC1C,IAA8C,EAC9C,OAAkC;IAElC,wBAAwB,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC5C,IAAI,wBAAwB,CAAC,IAAI,GAAG,4BAA4B,EAAE,CAAC;QAC/D,MAAM,GAAG,GAAG,wBAAwB,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;QACzD,wBAAwB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACzC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,oCAAoC,CAAC,UAAqC;IACtF,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC3B,mCAAmC;QACnC,IAAI,YAAY,GAAG,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtD,IAAI,YAAY;YAAE,OAAO,YAAY,CAAC;QAEtC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,YAAY,GAAG,EAAE,MAAM,EAAE,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC;QAClD,CAAC;aAAM,CAAC;YACJ,MAAM,EAAE,IAAI,EAAE,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC;YACzC,YAAY,GAAG,EAAE,MAAM,EAAE,YAAY,CAAC,IAAI,CAAC,EAAE,GAAG,cAAc,EAAE,CAAC;QACrE,CAAC;QAED,8BAA8B,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QAEnD,OAAO,YAAY,CAAC;IACxB,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,6BAA6B,CAAC,KAA2B;IACrE,OAAO,KAAK;SACP,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;QACb,uCAAuC;QACvC,IAAI,CAAC,IAAI,EAAE,CAAC;YACR,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;QAClC,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACZ,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;QACvC,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACV,8BAA8B;QAC9B,IAAI,UAAU,GAAG,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpD,IAAI,UAAU;YAAE,OAAO,UAAU,CAAC;QAElC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,UAAU,GAAG,EAAE,IAAI,EAAE,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC;QACrD,CAAC;aAAM,CAAC;YACJ,MAAM,EAAE,IAAI,EAAE,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC;YACzC,UAAU,GAAG,EAAE,IAAI,EAAE,mBAAmB,CAAC,IAAI,CAAC,EAAE,GAAG,cAAc,EAAE,CAAC;QACxE,CAAC;QAED,8BAA8B,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QAEjD,OAAO,UAAU,CAAC;IACtB,CAAC,CAAC,CAAC;AACX,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAAY;IAC5C,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAChC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,8BAA8B,WAAW,mCAAmC,CAAC,CAAC;IAClG,OAAO,WAAW,CAAC;AACvB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iCAAiC,CAAC,OAA+B;IAC7E,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,gCAAgC;QAChC,IAAI,YAAY,GAAG,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtD,IAAI,YAAY;YAAE,OAAO,YAAY,CAAC;QAEtC,IAAI,IAAI,YAAY,MAAM,EAAE,CAAC;YACzB,YAAY,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QACpC,CAAC;aAAM,CAAC;YACJ,YAAY,GAAG,IAAI,CAAC;QACxB,CAAC;QAED,8BAA8B,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QAEnD,OAAO,YAAY,CAAC;IACxB,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAC1B,cAA2C,EAC3C,iBAAsC,EACtC,wBAA4C,EAAE,EAC9C,QAA0C;IAE1C,OAAO,cAAc;SAChB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;SAC1E,MAAM,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;QAChB,OAAO,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC,oBAAoB,EAAE,EAAE;YACxD,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,oBAAoB,CAAC;YAC9C,OAAO,CAAC,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QAC7F,CAAC,CAAC,CAAC;IACP,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE;QACnB,IAAI,CAAC,iBAAiB,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,CAAC;YAClD,OAAO,IAAI,OAAO,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,eAAe,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC;QAC1G,CAAC;QAED,KAAK,MAAM,gBAAgB,IAAI,iBAAiB,EAAE,CAAC;YAC/C,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,oBAAoB,EAAE,GAAG,gBAAgB,CAAC;YACnE,IAAI,CAAC,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;gBACpF,MAAM,OAAO,GACT,OAAO,IAAI,KAAK,QAAQ;oBACpB,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,oBAAoB,EAAE,eAAe,EAAE,QAAQ,EAAE;oBACnE,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,GAAG,oBAAoB,EAAE,eAAe,EAAE,QAAQ,EAAE,CAAC;gBAE1E,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;YAChC,CAAC;QACL,CAAC;QAED,oCAAoC;QACpC,OAAO,IAAI,CAAC;IAChB,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAc,CAAC;AACnD,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,QAAmB,EAAE,QAA6B;IACvF,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;QACpB,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,MAAM,QAAQ,GAAc,EAAE,CAAC;IAE/B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC7B,KAAK,MAAM,gBAAgB,IAAI,QAAQ,EAAE,CAAC;YACtC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,gBAAgB,CAAC;YAE1C,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,SAAS,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;gBACpG,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACvB,kEAAkE;gBAClE,MAAM;YACV,CAAC;QACL,CAAC;IACL,CAAC;IAED,OAAO,QAAQ,CAAC;AACpB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAChC,OAAsD,EACtD,UAAuG,EAAE;IAEzG,OAAO,OAAO;SACT,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CACT,OAAO,GAAG,KAAK,QAAQ;QACnB,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,eAAe,EAAE,OAAO,CAAC,QAAQ,EAAE;QACjD,CAAC,CAAE,EAAE,GAAG,GAAG,EAAE,eAAe,EAAE,OAAO,CAAC,QAAQ,EAAqB,CAC1E;SACA,MAAM,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;QAChB,IAAI,CAAC;YACD,OAAO,IAAI,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC;QAC9C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,cAAc,EAAE,EAAE;QACpB,cAAc,CAAC,GAAG,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,GAAG,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC;QACvE,cAAc,CAAC,QAAQ,KAAK,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;QAEnD,IAAI,OAAO,OAAO,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;YACpC,cAAc,CAAC,QAAQ,GAAG;gBACtB,GAAG,cAAc,CAAC,QAAQ;gBAC1B,KAAK,EAAE,OAAO,CAAC,KAAK;aACvB,CAAC;QACN,CAAC;QAED,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YACzB,cAAc,CAAC,cAAc,GAAG,IAAI,CAAC;QACzC,CAAC;QAED,OAAO,cAAc,CAAC;IAC1B,CAAC,CAAC,CAAC;AACX,CAAC"}
+{"version":3,"file":"shared.js","sourceRoot":"","sources":["../../src/enqueue_links/shared.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAG/B,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEtC,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAGjD,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAGxC,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAEhD,MAAM,4BAA4B,GAAG,IAAI,CAAC;AAE1C;;;;;GAKG;AACH,MAAM,wBAAwB,GAAG,IAAI,GAAG,EAAE,CAAC;AAgC3C;;GAEG;AACH,MAAM,UAAU,8BAA8B,CAC1C,IAA8C,EAC9C,OAAkC;IAElC,wBAAwB,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC5C,IAAI,wBAAwB,CAAC,IAAI,GAAG,4BAA4B,EAAE,CAAC;QAC/D,MAAM,GAAG,GAAG,wBAAwB,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;QACzD,wBAAwB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACzC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,oCAAoC,CAAC,UAAqC;IACtF,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC3B,mCAAmC;QACnC,IAAI,YAAY,GAAG,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtD,IAAI,YAAY;YAAE,OAAO,YAAY,CAAC;QAEtC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,YAAY,GAAG,EAAE,MAAM,EAAE,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC;QAClD,CAAC;aAAM,CAAC;YACJ,MAAM,EAAE,IAAI,EAAE,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC;YACzC,YAAY,GAAG,EAAE,MAAM,EAAE,YAAY,CAAC,IAAI,CAAC,EAAE,GAAG,cAAc,EAAE,CAAC;QACrE,CAAC;QAED,8BAA8B,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QAEnD,OAAO,YAAY,CAAC;IACxB,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,6BAA6B,CAAC,KAA2B;IACrE,OAAO,KAAK;SACP,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;QACb,uCAAuC;QACvC,IAAI,CAAC,IAAI,EAAE,CAAC;YACR,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;QAClC,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACZ,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;QACvC,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACV,8BAA8B;QAC9B,IAAI,UAAU,GAAG,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpD,IAAI,UAAU;YAAE,OAAO,UAAU,CAAC;QAElC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,UAAU,GAAG,EAAE,IAAI,EAAE,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC;QACrD,CAAC;aAAM,CAAC;YACJ,MAAM,EAAE,IAAI,EAAE,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC;YACzC,UAAU,GAAG,EAAE,IAAI,EAAE,mBAAmB,CAAC,IAAI,CAAC,EAAE,GAAG,cAAc,EAAE,CAAC;QACxE,CAAC;QAED,8BAA8B,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QAEjD,OAAO,UAAU,CAAC;IACtB,CAAC,CAAC,CAAC;AACX,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAAY;IAC5C,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAChC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,8BAA8B,WAAW,mCAAmC,CAAC,CAAC;IAClG,OAAO,WAAW,CAAC;AACvB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iCAAiC,CAAC,OAA+B;IAC7E,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,gCAAgC;QAChC,IAAI,YAAY,GAAG,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtD,IAAI,YAAY;YAAE,OAAO,YAAY,CAAC;QAEtC,IAAI,IAAI,YAAY,MAAM,EAAE,CAAC;YACzB,YAAY,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QACpC,CAAC;aAAM,CAAC;YACJ,YAAY,GAAG,IAAI,CAAC;QACxB,CAAC;QAED,8BAA8B,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QAEnD,OAAO,YAAY,CAAC;IACxB,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAC1B,cAA2C,EAC3C,iBAAsC,EACtC,wBAA4C,EAAE,EAC9C,QAA0C,EAC1C,YAAoC;IAEpC,MAAM,4BAA4B,GAAG,qBAAqB,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;IAC3F,MAAM,wBAAwB,GAAG,iBAAiB,EAAE,GAAG,CAAC,0BAA0B,CAAC,CAAC;IAEpF,OAAO,cAAc;SAChB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;SAC1E,MAAM,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;QAChB,MAAM,sBAAsB,GAAG,4BAA4B,CAAC,IAAI,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;QAE5F,IAAI,sBAAsB,EAAE,CAAC;YACzB,YAAY,EAAE,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;QAED,OAAO,CAAC,sBAAsB,CAAC;IACnC,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE;QACnB,IAAI,CAAC,wBAAwB,IAAI,CAAC,wBAAwB,CAAC,MAAM,EAAE,CAAC;YAChE,OAAO,IAAI,OAAO,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,eAAe,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC;QAC1G,CAAC;QAED,KAAK,MAAM,gBAAgB,IAAI,wBAAwB,EAAE,CAAC;YACtD,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,oBAAoB,EAAE,GAAG,gBAAgB,CAAC;YAC1E,IAAI,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;gBACb,MAAM,OAAO,GACT,OAAO,IAAI,KAAK,QAAQ;oBACpB,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,oBAAoB,EAAE,eAAe,EAAE,QAAQ,EAAE;oBACnE,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,GAAG,oBAAoB,EAAE,eAAe,EAAE,QAAQ,EAAE,CAAC;gBAE1E,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;YAChC,CAAC;QACL,CAAC;QAED,oCAAoC;QACpC,YAAY,EAAE,CAAC,GAAG,CAAC,CAAC;QACpB,OAAO,IAAI,CAAC;IAChB,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAc,CAAC;AACnD,CAAC;AAED,MAAM,UAAU,wBAAwB,CACpC,QAAmB,EACnB,QAA6B,EAC7B,YAAoC;IAEpC,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;QACpB,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,MAAM,eAAe,GAAG,QAAQ,EAAE,GAAG,CAAC,0BAA0B,CAAC,CAAC;IAElE,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC7B,MAAM,eAAe,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;QAEhF,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;YAChC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3B,CAAC;aAAM,CAAC;YACJ,YAAY,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAChC,CAAC;IACL,CAAC;IAED,OAAO,QAAQ,CAAC;AACpB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAChC,OAAsD,EACtD,UAAuG,EAAE;IAEzG,OAAO,OAAO;SACT,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CACT,OAAO,GAAG,KAAK,QAAQ;QACnB,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,eAAe,EAAE,OAAO,CAAC,QAAQ,EAAE;QACjD,CAAC,CAAE,EAAE,GAAG,GAAG,EAAE,eAAe,EAAE,OAAO,CAAC,QAAQ,EAAqB,CAC1E;SACA,MAAM,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;QAChB,IAAI,CAAC;YACD,OAAO,IAAI,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC;QAC9C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,cAAc,EAAE,EAAE;QACpB,cAAc,CAAC,GAAG,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,GAAG,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC;QACvE,cAAc,CAAC,QAAQ,KAAK,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;QAEnD,IAAI,OAAO,OAAO,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;YACpC,cAAc,CAAC,QAAQ,GAAG;gBACtB,GAAG,cAAc,CAAC,QAAQ;gBAC1B,KAAK,EAAE,OAAO,CAAC,KAAK;aACvB,CAAC;QACN,CAAC;QAED,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YACzB,cAAc,CAAC,cAAc,GAAG,IAAI,CAAC;QACzC,CAAC;QAED,OAAO,cAAc,CAAC;IAC1B,CAAC,CAAC,CAAC;AACX,CAAC;AAED;;GAEG;AACH,SAAS,0BAA0B,CAAC,gBAAkC;IAClE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,gBAAgB,CAAC;IAC1C,IAAI,KAAK,CAAC;IACV,IAAI,MAAM,EAAE,CAAC;QACT,KAAK,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC9C,CAAC;SAAM,IAAI,IAAI,EAAE,CAAC;QACd,MAAM,CAAC,GAAG,IAAI,SAAS,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;QAChD,KAAK,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC1C,CAAC;SAAM,CAAC;QACJ,KAAK,GAAG,GAAG,EAAE,CAAC,KAAK,CAAC;IACxB,CAAC;IACD,OAAO,EAAE,GAAG,gBAAgB,EAAE,KAAK,EAAE,CAAC;AAC1C,CAAC"}

package/errors.d.ts

@@ -30,4 +30,16 @@ export declare class RetryRequestError extends Error {
 export declare class SessionError extends RetryRequestError {
     constructor(message?: string);
 }
+export declare class ContextPipelineInterruptedError extends Error {
+    constructor(message?: string);
+}
+export declare class ContextPipelineInitializationError extends Error {
+    constructor(error: unknown, options?: ErrorOptions);
+}
+export declare class ContextPipelineCleanupError extends CriticalError {
+    constructor(error: unknown, options?: ErrorOptions);
+}
+export declare class RequestHandlerError extends Error {
+    constructor(error: unknown, options?: ErrorOptions);
+}
 //# sourceMappingURL=errors.d.ts.map

package/errors.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,KAAK;CAAG;AAE/C;;;GAGG;AACH,qBAAa,aAAc,SAAQ,iBAAiB;CAAG;AAEvD;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,aAAa;CAAG;AAEvD;;;;GAIG;AACH,qBAAa,iBAAkB,SAAQ,KAAK;gBAC5B,OAAO,CAAC,EAAE,MAAM;CAG/B;AAED;;;;GAIG;AACH,qBAAa,YAAa,SAAQ,iBAAiB;gBACnC,OAAO,CAAC,EAAE,MAAM;CAG/B"}
+{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,KAAK;CAAG;AAE/C;;;GAGG;AACH,qBAAa,aAAc,SAAQ,iBAAiB;CAAG;AAEvD;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,aAAa;CAAG;AAEvD;;;;GAIG;AACH,qBAAa,iBAAkB,SAAQ,KAAK;gBAC5B,OAAO,CAAC,EAAE,MAAM;CAG/B;AAED;;;;GAIG;AACH,qBAAa,YAAa,SAAQ,iBAAiB;gBACnC,OAAO,CAAC,EAAE,MAAM;CAG/B;AAED,qBAAa,+BAAgC,SAAQ,KAAK;gBAC1C,OAAO,CAAC,EAAE,MAAM;CAG/B;AAED,qBAAa,kCAAmC,SAAQ,KAAK;gBAC7C,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,YAAY;CAGrD;AAED,qBAAa,2BAA4B,SAAQ,aAAa;gBAC9C,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,YAAY;CAGrD;AAED,qBAAa,mBAAoB,SAAQ,KAAK;gBAC9B,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,YAAY;CAGrD"}

package/errors.js

@@ -34,4 +34,24 @@ export class SessionError extends RetryRequestError {
         super(`Detected a session error, rotating session... ${message ? `\n${message}` : ''}`);
     }
 }
+export class ContextPipelineInterruptedError extends Error {
+    constructor(message) {
+        super(`Request handling was interrupted during context initialization ${message ? ` - ${message}` : ''}`);
+    }
+}
+export class ContextPipelineInitializationError extends Error {
+    constructor(error, options) {
+        super(undefined, { cause: error, ...options });
+    }
+}
+export class ContextPipelineCleanupError extends CriticalError {
+    constructor(error, options) {
+        super(undefined, { cause: error, ...options });
+    }
+}
+export class RequestHandlerError extends Error {
+    constructor(error, options) {
+        super(undefined, { cause: error, ...options });
+    }
+}
 //# sourceMappingURL=errors.js.map

package/errors.js.map

@@ -1 +1 @@
-{"version":3,"file":"errors.js","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,OAAO,iBAAkB,SAAQ,KAAK;CAAG;AAE/C;;;GAGG;AACH,MAAM,OAAO,aAAc,SAAQ,iBAAiB;CAAG;AAEvD;;GAEG;AACH,MAAM,OAAO,iBAAkB,SAAQ,aAAa;CAAG;AAEvD;;;;GAIG;AACH,MAAM,OAAO,iBAAkB,SAAQ,KAAK;IACxC,YAAY,OAAgB;QACxB,KAAK,CAAC,OAAO,IAAI,gDAAgD,CAAC,CAAC;IACvE,CAAC;CACJ;AAED;;;;GAIG;AACH,MAAM,OAAO,YAAa,SAAQ,iBAAiB;IAC/C,YAAY,OAAgB;QACxB,KAAK,CAAC,iDAAiD,OAAO,CAAC,CAAC,CAAC,KAAK,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC5F,CAAC;CACJ"}
+{"version":3,"file":"errors.js","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,OAAO,iBAAkB,SAAQ,KAAK;CAAG;AAE/C;;;GAGG;AACH,MAAM,OAAO,aAAc,SAAQ,iBAAiB;CAAG;AAEvD;;GAEG;AACH,MAAM,OAAO,iBAAkB,SAAQ,aAAa;CAAG;AAEvD;;;;GAIG;AACH,MAAM,OAAO,iBAAkB,SAAQ,KAAK;IACxC,YAAY,OAAgB;QACxB,KAAK,CAAC,OAAO,IAAI,gDAAgD,CAAC,CAAC;IACvE,CAAC;CACJ;AAED;;;;GAIG;AACH,MAAM,OAAO,YAAa,SAAQ,iBAAiB;IAC/C,YAAY,OAAgB;QACxB,KAAK,CAAC,iDAAiD,OAAO,CAAC,CAAC,CAAC,KAAK,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC5F,CAAC;CACJ;AAED,MAAM,OAAO,+BAAgC,SAAQ,KAAK;IACtD,YAAY,OAAgB;QACxB,KAAK,CAAC,kEAAkE,OAAO,CAAC,CAAC,CAAC,MAAM,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC9G,CAAC;CACJ;AAED,MAAM,OAAO,kCAAmC,SAAQ,KAAK;IACzD,YAAY,KAAc,EAAE,OAAsB;QAC9C,KAAK,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACnD,CAAC;CACJ;AAED,MAAM,OAAO,2BAA4B,SAAQ,aAAa;IAC1D,YAAY,KAAc,EAAE,OAAsB;QAC9C,KAAK,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACnD,CAAC;CACJ;AAED,MAAM,OAAO,mBAAoB,SAAQ,KAAK;IAC1C,YAAY,KAAc,EAAE,OAAsB;QAC9C,KAAK,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACnD,CAAC;CACJ"}

package/events/local_event_manager.d.ts

@@ -1,6 +1,5 @@
 import { EventManager } from './event_manager.js';
 export declare class LocalEventManager extends EventManager {
-    private previousTicks;
     /**
      * Initializes the EventManager and sets up periodic `systemInfo` and `persistState` events.
      * This is automatically called at the beginning of `crawler.run()`.
@@ -18,7 +17,6 @@ export declare class LocalEventManager extends EventManager {
      * @internal
      */
     isContainerizedWrapper(): Promise<boolean>;
-    private getCurrentCpuTicks;
     /**
      * Creates a SystemInfo object based on local metrics.
      */

package/events/local_event_manager.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"local_event_manager.d.ts","sourceRoot":"","sources":["../../src/events/local_event_manager.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,YAAY,EAAa,MAAM,oBAAoB,CAAC;AAE7D,qBAAa,iBAAkB,SAAQ,YAAY;IAC/C,OAAO,CAAC,aAAa,CAAyB;IAE9C;;;OAGG;IACY,IAAI;IAYnB;;OAEG;IACY,KAAK;IASpB;;OAEG;IACG,mBAAmB,CAAC,gBAAgB,EAAE,MAAM,OAAO;IAQzD;;OAEG;IACG,sBAAsB;IAI5B,OAAO,CAAC,kBAAkB;IAc1B;;OAEG;YACW,gBAAgB;YAQhB,aAAa;YAoBb,gBAAgB;CAiBjC"}
+{"version":3,"file":"local_event_manager.d.ts","sourceRoot":"","sources":["../../src/events/local_event_manager.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,YAAY,EAAa,MAAM,oBAAoB,CAAC;AAE7D,qBAAa,iBAAkB,SAAQ,YAAY;IAC/C;;;OAGG;IACY,IAAI;IAYnB;;OAEG;IACY,KAAK;IASpB;;OAEG;IACG,mBAAmB,CAAC,gBAAgB,EAAE,MAAM,OAAO;IAQzD;;OAEG;IACG,sBAAsB;IAI5B;;OAEG;YACW,gBAAgB;YAQhB,aAAa;YAQb,gBAAgB;CAWjC"}

package/events/local_event_manager.js

@@ -1,10 +1,8 @@
-import os from 'node:os';
-import { getCurrentCpuTicksV2, getMemoryInfo, getMemoryInfoV2, isContainerized } from '@crawlee/utils';
+import { getCurrentCpuTicksV2, getMemoryInfo, isContainerized } from '@crawlee/utils';
 import log from '@apify/log';
 import { betterClearInterval, betterSetInterval } from '@apify/utilities';
 import { EventManager } from './event_manager.js';
 export class LocalEventManager extends EventManager {
-    previousTicks = { idle: 0, total: 0 };
     /**
      * Initializes the EventManager and sets up periodic `systemInfo` and `persistState` events.
      * This is automatically called at the beginning of `crawler.run()`.
@@ -44,16 +42,6 @@ export class LocalEventManager extends EventManager {
     async isContainerizedWrapper() {
         return this.config.get('containerized', await isContainerized());
     }
-    getCurrentCpuTicks() {
-        const cpus = os.cpus();
-        return cpus.reduce((acc, cpu) => {
-            const cpuTimes = Object.values(cpu.times);
-            return {
-                idle: acc.idle + cpu.times.idle,
-                total: acc.total + cpuTimes.reduce((sum, num) => sum + num),
-            };
-        }, { idle: 0, total: 0 });
-    }
     /**
      * Creates a SystemInfo object based on local metrics.
      */
@@ -65,18 +53,7 @@ export class LocalEventManager extends EventManager {
         };
     }
     async createCpuInfo(options) {
-        if (this.config.get('systemInfoV2')) {
-            const usedCpuRatio = await getCurrentCpuTicksV2(await this.isContainerizedWrapper());
-            return {
-                cpuCurrentUsage: usedCpuRatio * 100,
-                isCpuOverloaded: usedCpuRatio > options.maxUsedCpuRatio,
-            };
-        }
-        const ticks = this.getCurrentCpuTicks();
-        const idleTicksDelta = ticks.idle - this.previousTicks.idle;
-        const totalTicksDelta = ticks.total - this.previousTicks.total;
-        const usedCpuRatio = totalTicksDelta ? 1 - idleTicksDelta / totalTicksDelta : 0;
-        Object.assign(this.previousTicks, ticks);
+        const usedCpuRatio = await getCurrentCpuTicksV2(await this.isContainerizedWrapper());
         return {
             cpuCurrentUsage: usedCpuRatio * 100,
             isCpuOverloaded: usedCpuRatio > options.maxUsedCpuRatio,
@@ -84,13 +61,7 @@ export class LocalEventManager extends EventManager {
     }
     async createMemoryInfo() {
         try {
-            if (this.config.get('systemInfoV2')) {
-                const memInfo = await getMemoryInfoV2(await this.isContainerizedWrapper());
-                return {
-                    memCurrentBytes: memInfo.mainProcessBytes + memInfo.childProcessesBytes,
-                };
-            }
-            const memInfo = await getMemoryInfo();
+            const memInfo = await getMemoryInfo(await this.isContainerizedWrapper());
             return {
                 memCurrentBytes: memInfo.mainProcessBytes + memInfo.childProcessesBytes,
             };

package/events/local_event_manager.js.map

@@ -1 +1 @@
-{"version":3,"file":"local_event_manager.js","sourceRoot":"","sources":["../../src/events/local_event_manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AAEzB,OAAO,EAAE,oBAAoB,EAAE,aAAa,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAEvG,OAAO,GAAG,MAAM,YAAY,CAAC;AAC7B,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAG1E,OAAO,EAAE,YAAY,EAAa,MAAM,oBAAoB,CAAC;AAE7D,MAAM,OAAO,iBAAkB,SAAQ,YAAY;IACvC,aAAa,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;IAE9C;;;OAGG;IACM,KAAK,CAAC,IAAI;QACf,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,OAAO;QACX,CAAC;QAED,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QAEnB,MAAM,wBAAwB,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,0BAA0B,CAAE,CAAC;QAC9E,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/D,IAAI,CAAC,SAAS,CAAC,UAAU,GAAG,iBAAiB,CAAC,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,wBAAwB,CAAC,CAAC;IACjH,CAAC;IAED;;OAEG;IACM,KAAK,CAAC,KAAK;QAChB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACpB,OAAO;QACX,CAAC;QAED,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;QACpB,mBAAmB,CAAC,IAAI,CAAC,SAAS,CAAC,UAAW,CAAC,CAAC;IACpD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,mBAAmB,CAAC,gBAA+B;QACrD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC;YACrC,eAAe,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,iBAAiB,CAAC;SACtD,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,CAAC,IAAI,2CAAwB,IAAI,CAAC,CAAC;QAC9C,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,sBAAsB;QACxB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,eAAe,EAAE,MAAM,eAAe,EAAE,CAAC,CAAC;IACrE,CAAC;IAEO,kBAAkB;QACtB,MAAM,IAAI,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC,MAAM,CACd,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;YACT,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAC1C,OAAO;gBACH,IAAI,EAAE,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI;gBAC/B,KAAK,EAAE,GAAG,CAAC,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC;aAC9D,CAAC;QACN,CAAC,EACD,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CACxB,CAAC;IACN,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,gBAAgB,CAAC,OAAoC;QAC/D,OAAO;YACH,SAAS,EAAE,IAAI,IAAI,EAAE;YACrB,GAAG,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;YACtC,GAAG,CAAC,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;SACvB,CAAC;IACpB,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAAoC;QAC5D,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;YAClC,MAAM,YAAY,GAAG,MAAM,oBAAoB,CAAC,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC,CAAC;YACrF,OAAO;gBACH,eAAe,EAAE,YAAY,GAAG,GAAG;gBACnC,eAAe,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe;aAC1D,CAAC;QACN,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACxC,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,aAAc,CAAC,IAAI,CAAC;QAC7D,MAAM,eAAe,GAAG,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,aAAc,CAAC,KAAK,CAAC;QAChE,MAAM,YAAY,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,GAAG,cAAc,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC;QAChF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;QAEzC,OAAO;YACH,eAAe,EAAE,YAAY,GAAG,GAAG;YACnC,eAAe,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe;SAC1D,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,gBAAgB;QAC1B,IAAI,CAAC;YACD,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;gBAClC,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC,CAAC;gBAC3E,OAAO;oBACH,eAAe,EAAE,OAAO,CAAC,gBAAgB,GAAG,OAAO,CAAC,mBAAmB;iBAC1E,CAAC;YACN,CAAC;YACD,MAAM,OAAO,GAAG,MAAM,aAAa,EAAE,CAAC;YACtC,OAAO;gBACH,eAAe,EAAE,OAAO,CAAC,gBAAgB,GAAG,OAAO,CAAC,mBAAmB;aAC1E,CAAC;QACN,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,GAAG,CAAC,SAAS,CAAC,GAAY,EAAE,yBAAyB,CAAC,CAAC;YACvD,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;CACJ"}
+{"version":3,"file":"local_event_manager.js","sourceRoot":"","sources":["../../src/events/local_event_manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAEtF,OAAO,GAAG,MAAM,YAAY,CAAC;AAC7B,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAG1E,OAAO,EAAE,YAAY,EAAa,MAAM,oBAAoB,CAAC;AAE7D,MAAM,OAAO,iBAAkB,SAAQ,YAAY;IAC/C;;;OAGG;IACM,KAAK,CAAC,IAAI;QACf,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,OAAO;QACX,CAAC;QAED,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QAEnB,MAAM,wBAAwB,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,0BAA0B,CAAE,CAAC;QAC9E,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/D,IAAI,CAAC,SAAS,CAAC,UAAU,GAAG,iBAAiB,CAAC,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,wBAAwB,CAAC,CAAC;IACjH,CAAC;IAED;;OAEG;IACM,KAAK,CAAC,KAAK;QAChB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACpB,OAAO;QACX,CAAC;QAED,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;QACpB,mBAAmB,CAAC,IAAI,CAAC,SAAS,CAAC,UAAW,CAAC,CAAC;IACpD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,mBAAmB,CAAC,gBAA+B;QACrD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC;YACrC,eAAe,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,iBAAiB,CAAC;SACtD,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,CAAC,IAAI,2CAAwB,IAAI,CAAC,CAAC;QAC9C,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,sBAAsB;QACxB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,eAAe,EAAE,MAAM,eAAe,EAAE,CAAC,CAAC;IACrE,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,gBAAgB,CAAC,OAAoC;QAC/D,OAAO;YACH,SAAS,EAAE,IAAI,IAAI,EAAE;YACrB,GAAG,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;YACtC,GAAG,CAAC,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;SACvB,CAAC;IACpB,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAAoC;QAC5D,MAAM,YAAY,GAAG,MAAM,oBAAoB,CAAC,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC,CAAC;QACrF,OAAO;YACH,eAAe,EAAE,YAAY,GAAG,GAAG;YACnC,eAAe,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe;SAC1D,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,gBAAgB;QAC1B,IAAI,CAAC;YACD,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC,CAAC;YACzE,OAAO;gBACH,eAAe,EAAE,OAAO,CAAC,gBAAgB,GAAG,OAAO,CAAC,mBAAmB;aAC1E,CAAC;QACN,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,GAAG,CAAC,SAAS,CAAC,GAAY,EAAE,yBAAyB,CAAC,CAAC;YACvD,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;CACJ"}

package/http_clients/base-http-client.d.ts

@@ -1,134 +1,12 @@
-import type { Readable } from 'node:stream';
-import { type SearchParams } from '@crawlee/utils';
-import type { FormDataLike } from './form-data-like.js';
-type Timeout = {
-    lookup: number;
-    connect: number;
-    secureConnect: number;
-    socket: number;
-    send: number;
-    response: number;
-} | {
-    request: number;
-};
-type Method = 'GET' | 'POST' | 'PUT' | 'PATCH' | 'HEAD' | 'DELETE' | 'OPTIONS' | 'TRACE' | 'get' | 'post' | 'put' | 'patch' | 'head' | 'delete' | 'options' | 'trace';
-/**
- * Maps permitted values of the `responseType` option on {@link HttpRequest} to the types that they produce.
- */
-export interface ResponseTypes {
-    'json': unknown;
-    'text': string;
-    'buffer': Buffer;
-}
-interface Progress {
-    percent: number;
-    transferred: number;
-    total?: number;
-}
-interface ToughCookieJar {
-    getCookieString: ((currentUrl: string, options: Record<string, unknown>, callback: (error: Error | null, cookies: string) => void) => string) & ((url: string, callback: (error: Error | null, cookieHeader: string) => void) => string);
-    setCookie: ((cookieOrString: unknown, currentUrl: string, options: Record<string, unknown>, callback: (error: Error | null, cookie: unknown) => void) => void) & ((rawCookie: string, url: string, callback: (error: Error | null, result: unknown) => void) => void);
-}
-interface PromiseCookieJar {
-    getCookieString: (url: string) => Promise<string>;
-    setCookie: (rawCookie: string, url: string) => Promise<unknown>;
-}
-type SimpleHeaders = Record<string, string | string[] | undefined>;
-/**
- * HTTP Request as accepted by {@link BaseHttpClient} methods.
- */
-export interface HttpRequest<TResponseType extends keyof ResponseTypes = 'text'> {
-    [k: string]: unknown;
-    url: string | URL;
-    method?: Method;
-    headers?: SimpleHeaders;
-    body?: string | Buffer | Readable | Generator | AsyncGenerator | FormDataLike;
-    signal?: AbortSignal;
-    timeout?: Partial<Timeout>;
-    cookieJar?: ToughCookieJar | PromiseCookieJar;
-    followRedirect?: boolean | ((response: any) => boolean);
-    maxRedirects?: number;
-    encoding?: BufferEncoding;
-    responseType?: TResponseType;
-    throwHttpErrors?: boolean;
-    proxyUrl?: string;
-    headerGeneratorOptions?: Record<string, unknown>;
-    useHeaderGenerator?: boolean;
-    headerGenerator?: {
-        getHeaders: (options: Record<string, unknown>) => Record<string, string>;
-    };
-    insecureHTTPParser?: boolean;
-    sessionToken?: object;
-}
-/**
- * Additional options for HTTP requests that need to be handled separately before passing to {@link BaseHttpClient}.
- */
-export interface HttpRequestOptions<TResponseType extends keyof ResponseTypes = 'text'> extends HttpRequest<TResponseType> {
-    /** Search (query string) parameters to be appended to the request URL */
-    searchParams?: SearchParams;
-    /** A form to be sent in the HTTP request body (URL encoding will be used) */
-    form?: Record<string, string>;
-    /** Artbitrary object to be JSON-serialized and sent as the HTTP request body */
-    json?: unknown;
-    /** Basic HTTP Auth username */
-    username?: string;
-    /** Basic HTTP Auth password */
-    password?: string;
-}
-/**
- * HTTP response data, without a body, as returned by {@link BaseHttpClient} methods.
- */
-export interface BaseHttpResponseData {
-    redirectUrls: URL[];
+import type { HttpRequest, HttpRequestOptions, IResponseWithUrl } from '@crawlee/types';
+export declare class ResponseWithUrl extends Response implements IResponseWithUrl {
     url: string;
-    ip?: string;
-    statusCode: number;
-    statusMessage?: string;
-    headers: SimpleHeaders;
-    trailers: SimpleHeaders;
-    complete: boolean;
-}
-interface HttpResponseWithoutBody<TResponseType extends keyof ResponseTypes = keyof ResponseTypes> extends BaseHttpResponseData {
-    request: HttpRequest<TResponseType>;
-}
-/**
- * HTTP response data as returned by the {@link BaseHttpClient.sendRequest} method.
- */
-export interface HttpResponse<TResponseType extends keyof ResponseTypes = keyof ResponseTypes> extends HttpResponseWithoutBody<TResponseType> {
-    [k: string]: any;
-    body: ResponseTypes[TResponseType];
-}
-/**
- * HTTP response data as returned by the {@link BaseHttpClient.stream} method.
- */
-export interface StreamingHttpResponse extends HttpResponseWithoutBody {
-    stream: Readable;
-    readonly downloadProgress: Progress;
-    readonly uploadProgress: Progress;
-}
-/**
- * Type of a function called when an HTTP redirect takes place. It is allowed to mutate the `updatedRequest` argument.
- */
-export type RedirectHandler = (redirectResponse: BaseHttpResponseData, updatedRequest: {
-    url?: string | URL;
-    headers: SimpleHeaders;
-}) => void;
-/**
- * Interface for user-defined HTTP clients to be used for plain HTTP crawling and for sending additional requests during a crawl.
- */
-export interface BaseHttpClient {
-    /**
-     * Perform an HTTP Request and return the complete response.
-     */
-    sendRequest<TResponseType extends keyof ResponseTypes = 'text'>(request: HttpRequest<TResponseType>): Promise<HttpResponse<TResponseType>>;
-    /**
-     * Perform an HTTP Request and return after the response headers are received. The body may be read from a stream contained in the response.
-     */
-    stream(request: HttpRequest, onRedirect?: RedirectHandler): Promise<StreamingHttpResponse>;
+    constructor(body: BodyInit | null, init: ResponseInit & {
+        url?: string;
+    });
 }
 /**
  * Converts {@link HttpRequestOptions} to a {@link HttpRequest}.
  */
-export declare function processHttpRequestOptions<TResponseType extends keyof ResponseTypes = 'text'>({ searchParams, form, json, username, password, ...request }: HttpRequestOptions<TResponseType>): HttpRequest<TResponseType>;
-export {};
+export declare function processHttpRequestOptions({ searchParams, form, json, username, password, ...request }: HttpRequestOptions): HttpRequest;
 //# sourceMappingURL=base-http-client.d.ts.map

package/http_clients/base-http-client.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"base-http-client.d.ts","sourceRoot":"","sources":["../../src/http_clients/base-http-client.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAE5C,OAAO,EAAqB,KAAK,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAEtE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAExD,KAAK,OAAO,GACN;IACI,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;CACpB,GACD;IAAE,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC;AAE1B,KAAK,MAAM,GACL,KAAK,GACL,MAAM,GACN,KAAK,GACL,OAAO,GACP,MAAM,GACN,QAAQ,GACR,SAAS,GACT,OAAO,GACP,KAAK,GACL,MAAM,GACN,KAAK,GACL,OAAO,GACP,MAAM,GACN,QAAQ,GACR,SAAS,GACT,OAAO,CAAC;AAEd;;GAEG;AACH,MAAM,WAAW,aAAa;IAC1B,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;CACpB;AAED,UAAU,QAAQ;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAGD,UAAU,cAAc;IACpB,eAAe,EAAE,CAAC,CACd,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAChC,QAAQ,EAAE,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,KACvD,MAAM,CAAC,GACR,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,EAAE,YAAY,EAAE,MAAM,KAAK,IAAI,KAAK,MAAM,CAAC,CAAC;IAC7F,SAAS,EAAE,CAAC,CACR,cAAc,EAAE,OAAO,EACvB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAChC,QAAQ,EAAE,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,EAAE,MAAM,EAAE,OAAO,KAAK,IAAI,KACvD,IAAI,CAAC,GACN,CAAC,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,EAAE,MAAM,EAAE,OAAO,KAAK,IAAI,KAAK,IAAI,CAAC,CAAC;CAC5G;AAED,UAAU,gBAAgB;IACtB,eAAe,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IAClD,SAAS,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;CACnE;AAED,KAAK,aAAa,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,SAAS,CAAC,CAAC;AAEnE;;GAEG;AACH,MAAM,WAAW,WAAW,CAAC,aAAa,SAAS,MAAM,aAAa,GAAG,MAAM;IAC3E,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IAErB,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,aAAa,CAAC;IACxB,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,QAAQ,GAAG,SAAS,GAAG,cAAc,GAAG,YAAY,CAAC;IAE9E,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,OAAO,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IAE3B,SAAS,CAAC,EAAE,cAAc,GAAG,gBAAgB,CAAC;IAC9C,cAAc,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,QAAQ,EAAE,GAAG,KAAK,OAAO,CAAC,CAAC;IACxD,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB,QAAQ,CAAC,EAAE,cAAc,CAAC;IAC1B,YAAY,CAAC,EAAE,aAAa,CAAC;IAC7B,eAAe,CAAC,EAAE,OAAO,CAAC;IAG1B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sBAAsB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjD,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,eAAe,CAAC,EAAE;QACd,UAAU,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAC5E,CAAC;IACF,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB,CAAC,aAAa,SAAS,MAAM,aAAa,GAAG,MAAM,CAClF,SAAQ,WAAW,CAAC,aAAa,CAAC;IAClC,yEAAyE;IACzE,YAAY,CAAC,EAAE,YAAY,CAAC;IAE5B,6EAA6E;IAC7E,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9B,gFAAgF;IAChF,IAAI,CAAC,EAAE,OAAO,CAAC;IAEf,+BAA+B;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,+BAA+B;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACjC,YAAY,EAAE,GAAG,EAAE,CAAC;IACpB,GAAG,EAAE,MAAM,CAAC;IAEZ,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB,OAAO,EAAE,aAAa,CAAC;IACvB,QAAQ,EAAE,aAAa,CAAC;IAExB,QAAQ,EAAE,OAAO,CAAC;CACrB;AAED,UAAU,uBAAuB,CAAC,aAAa,SAAS,MAAM,aAAa,GAAG,MAAM,aAAa,CAC7F,SAAQ,oBAAoB;IAC5B,OAAO,EAAE,WAAW,CAAC,aAAa,CAAC,CAAC;CACvC;AAED;;GAEG;AACH,MAAM,WAAW,YAAY,CAAC,aAAa,SAAS,MAAM,aAAa,GAAG,MAAM,aAAa,CACzF,SAAQ,uBAAuB,CAAC,aAAa,CAAC;IAC9C,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAC;IAEjB,IAAI,EAAE,aAAa,CAAC,aAAa,CAAC,CAAC;CACtC;AAED;;GAEG;AACH,MAAM,WAAW,qBAAsB,SAAQ,uBAAuB;IAClE,MAAM,EAAE,QAAQ,CAAC;IACjB,QAAQ,CAAC,gBAAgB,EAAE,QAAQ,CAAC;IACpC,QAAQ,CAAC,cAAc,EAAE,QAAQ,CAAC;CACrC;AAED;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,CAC1B,gBAAgB,EAAE,oBAAoB,EACtC,cAAc,EAAE;IAAE,GAAG,CAAC,EAAE,MAAM,GAAG,GAAG,CAAC;IAAC,OAAO,EAAE,aAAa,CAAA;CAAE,KAC7D,IAAI,CAAC;AAEV;;GAEG;AACH,MAAM,WAAW,cAAc;IAC3B;;OAEG;IACH,WAAW,CAAC,aAAa,SAAS,MAAM,aAAa,GAAG,MAAM,EAC1D,OAAO,EAAE,WAAW,CAAC,aAAa,CAAC,GACpC,OAAO,CAAC,YAAY,CAAC,aAAa,CAAC,CAAC,CAAC;IAExC;;OAEG;IACH,MAAM,CAAC,OAAO,EAAE,WAAW,EAAE,UAAU,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAAC;CAC9F;AAED;;GAEG;AACH,wBAAgB,yBAAyB,CAAC,aAAa,SAAS,MAAM,aAAa,GAAG,MAAM,EAAE,EAC1F,YAAY,EACZ,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,GAAG,OAAO,EACb,EAAE,kBAAkB,CAAC,aAAa,CAAC,GAAG,WAAW,CAAC,aAAa,CAAC,CAoChE"}
+{"version":3,"file":"base-http-client.d.ts","sourceRoot":"","sources":["../../src/http_clients/base-http-client.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,WAAW,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAGxF,qBAAa,eAAgB,SAAQ,QAAS,YAAW,gBAAgB;IAC5D,GAAG,EAAE,MAAM,CAAC;gBACT,IAAI,EAAE,QAAQ,GAAG,IAAI,EAAE,IAAI,EAAE,YAAY,GAAG;QAAE,GAAG,CAAC,EAAE,MAAM,CAAA;KAAE;CAI3E;AAED;;GAEG;AACH,wBAAgB,yBAAyB,CAAC,EACtC,YAAY,EACZ,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,GAAG,OAAO,EACb,EAAE,kBAAkB,GAAG,WAAW,CAwClC"}

package/http_clients/base-http-client.js

@@ -1,32 +1,43 @@
+import { Readable } from 'node:stream';
 import { applySearchParams } from '@crawlee/utils';
+export class ResponseWithUrl extends Response {
+    url;
+    constructor(body, init) {
+        super(body, init);
+        this.url = init.url ?? '';
+    }
+}
 /**
  * Converts {@link HttpRequestOptions} to a {@link HttpRequest}.
  */
 export function processHttpRequestOptions({ searchParams, form, json, username, password, ...request }) {
     const url = new URL(request.url);
-    const headers = { ...request.headers };
+    const headers = new Headers(request.headers);
     applySearchParams(url, searchParams);
     if ([request.body, form, json].filter((value) => value !== undefined).length > 1) {
         throw new Error('At most one of `body`, `form` and `json` may be specified in sendRequest arguments');
     }
     const body = (() => {
         if (form !== undefined) {
-            return new URLSearchParams(form).toString();
+            return Readable.from(new URLSearchParams(form).toString());
         }
         if (json !== undefined) {
-            return JSON.stringify(json);
+            return Readable.from(JSON.stringify(json));
+        }
+        if (request.body !== undefined) {
+            return Readable.from(request.body);
         }
-        return request.body;
+        return undefined;
     })();
-    if (form !== undefined) {
-        headers['content-type'] ??= 'application/x-www-form-urlencoded';
+    if (form !== undefined && !headers.has('content-type')) {
+        headers.set('content-type', 'application/x-www-form-urlencoded');
     }
-    if (json !== undefined) {
-        headers['content-type'] ??= 'application/json';
+    if (json !== undefined && !headers.has('content-type')) {
+        headers.set('content-type', 'application/json');
     }
     if (username !== undefined || password !== undefined) {
         const encodedAuth = Buffer.from(`${username ?? ''}:${password ?? ''}`).toString('base64');
-        headers.authorization = `Basic ${encodedAuth}`;
+        headers.set('authorization', `Basic ${encodedAuth}`);
     }
     return { ...request, body, url, headers };
 }

package/http_clients/base-http-client.js.map

@@ -1 +1 @@
-{"version":3,"file":"base-http-client.js","sourceRoot":"","sources":["../../src/http_clients/base-http-client.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,iBAAiB,EAAqB,MAAM,gBAAgB,CAAC;AA8LtE;;GAEG;AACH,MAAM,UAAU,yBAAyB,CAAqD,EAC1F,YAAY,EACZ,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,GAAG,OAAO,EACsB;IAChC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACjC,MAAM,OAAO,GAAG,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAEvC,iBAAiB,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IAErC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/E,MAAM,IAAI,KAAK,CAAC,oFAAoF,CAAC,CAAC;IAC1G,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,GAAG,EAAE;QACf,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACrB,OAAO,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;QAChD,CAAC;QAED,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACrB,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAChC,CAAC;QAED,OAAO,OAAO,CAAC,IAAI,CAAC;IACxB,CAAC,CAAC,EAAE,CAAC;IAEL,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACrB,OAAO,CAAC,cAAc,CAAC,KAAK,mCAAmC,CAAC;IACpE,CAAC;IAED,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACrB,OAAO,CAAC,cAAc,CAAC,KAAK,kBAAkB,CAAC;IACnD,CAAC;IAED,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QACnD,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,IAAI,EAAE,IAAI,QAAQ,IAAI,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC1F,OAAO,CAAC,aAAa,GAAG,SAAS,WAAW,EAAE,CAAC;IACnD,CAAC;IAED,OAAO,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC;AAC9C,CAAC"}
+{"version":3,"file":"base-http-client.js","sourceRoot":"","sources":["../../src/http_clients/base-http-client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAGvC,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAEnD,MAAM,OAAO,eAAgB,SAAQ,QAAQ;IAChC,GAAG,CAAS;IACrB,YAAY,IAAqB,EAAE,IAAqC;QACpE,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAClB,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC;IAC9B,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,UAAU,yBAAyB,CAAC,EACtC,YAAY,EACZ,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,GAAG,OAAO,EACO;IACjB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACjC,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAE7C,iBAAiB,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IAErC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/E,MAAM,IAAI,KAAK,CAAC,oFAAoF,CAAC,CAAC;IAC1G,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,GAAG,EAAE;QACf,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACrB,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC/D,CAAC;QAED,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACrB,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/C,CAAC;QAED,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC7B,OAAO,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACvC,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC,CAAC,EAAE,CAAC;IAEL,IAAI,IAAI,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,mCAAmC,CAAC,CAAC;IACrE,CAAC;IAED,IAAI,IAAI,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,kBAAkB,CAAC,CAAC;IACpD,CAAC;IAED,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QACnD,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,IAAI,EAAE,IAAI,QAAQ,IAAI,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC1F,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,SAAS,WAAW,EAAE,CAAC,CAAC;IACzD,CAAC;IAED,OAAO,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC;AAC9C,CAAC"}

package/http_clients/got-scraping-http-client.d.ts

@@ -1,15 +1,22 @@
-import type { BaseHttpClient, HttpRequest, HttpResponse, RedirectHandler, ResponseTypes, StreamingHttpResponse } from './base-http-client.js';
+import type { BaseHttpClient, SendRequestOptions, StreamOptions } from '@crawlee/types';
 /**
  * A HTTP client implementation based on the `got-scraping` library.
  */
 export declare class GotScrapingHttpClient implements BaseHttpClient {
+    /**
+     * Type guard that validates the HTTP method (excluding CONNECT).
+     * @param request - The HTTP request to validate
+     */
+    private validateRequest;
+    private iterateHeaders;
+    private parseHeaders;
     /**
      * @inheritDoc
      */
-    sendRequest<TResponseType extends keyof ResponseTypes>(request: HttpRequest<TResponseType>): Promise<HttpResponse<TResponseType>>;
+    sendRequest(request: Request, options?: SendRequestOptions): Promise<Response>;
     /**
      * @inheritDoc
      */
-    stream(request: HttpRequest, handleRedirect?: RedirectHandler): Promise<StreamingHttpResponse>;
+    stream(request: Request, options?: StreamOptions): Promise<Response>;
 }
 //# sourceMappingURL=got-scraping-http-client.d.ts.map

package/http_clients/got-scraping-http-client.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"got-scraping-http-client.d.ts","sourceRoot":"","sources":["../../src/http_clients/got-scraping-http-client.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACR,cAAc,EACd,WAAW,EACX,YAAY,EACZ,eAAe,EACf,aAAa,EACb,qBAAqB,EACxB,MAAM,uBAAuB,CAAC;AAE/B;;GAEG;AACH,qBAAa,qBAAsB,YAAW,cAAc;IACxD;;OAEG;IACG,WAAW,CAAC,aAAa,SAAS,MAAM,aAAa,EACvD,OAAO,EAAE,WAAW,CAAC,aAAa,CAAC,GACpC,OAAO,CAAC,YAAY,CAAC,aAAa,CAAC,CAAC;IAgBvC;;OAEG;IACG,MAAM,CAAC,OAAO,EAAE,WAAW,EAAE,cAAc,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,qBAAqB,CAAC;CAmDvG"}
+{"version":3,"file":"got-scraping-http-client.d.ts","sourceRoot":"","sources":["../../src/http_clients/got-scraping-http-client.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAMxF;;GAEG;AACH,qBAAa,qBAAsB,YAAW,cAAc;IACxD;;;OAGG;IACH,OAAO,CAAC,eAAe;IAMvB,OAAO,CAAE,cAAc;IAmBvB,OAAO,CAAC,YAAY;IAIpB;;OAEG;IACG,WAAW,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,QAAQ,CAAC;IA8BpF;;OAEG;IACG,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,QAAQ,CAAC;CA8D7E"}