@crawlee/core 4.0.0-beta.7 → 4.0.0-beta.71

package/enqueue_links/enqueue_links.js

@@ -1,7 +1,8 @@
 import ow from 'ow';
 import { getDomain } from 'tldts';
-import log from '@apify/log';
-import { constructGlobObjectsFromGlobs, constructRegExpObjectsFromPseudoUrls, constructRegExpObjectsFromRegExps, createRequestOptions, createRequests, filterRequestsByPatterns, } from './shared.js';
+import { Request } from '../request.js';
+import { serviceLocator } from '../service_locator.js';
+import { applyRequestTransform, constructGlobObjectsFromGlobs, constructRegExpObjectsFromPseudoUrls, constructRegExpObjectsFromRegExps, createRequestOptions, filterRequestOptionsByPatterns, } from './shared.js';
 /**
  * The different enqueueing strategies available.
  *
@@ -66,7 +67,7 @@ export var EnqueueStrategy;
  * ```javascript
  * await enqueueLinks({
  *   urls: aListOfFoundUrls,
- *   requestQueue,
+ *   requestManager,
  *   selector: 'a.product-detail',
  *   globs: [
  *       'https://www.example.com/handbags/*',
@@ -87,11 +88,12 @@ export async function enqueueLinks(options) {
     }
     ow(options, ow.object.exactShape({
         urls: ow.array.ofType(ow.string),
-        requestQueue: ow.object.hasKeys('fetchNextRequest', 'addRequest'),
+        requestManager: ow.object.hasKeys('addRequestsBatched'),
         robotsTxtFile: ow.optional.object.hasKeys('isAllowed'),
         onSkippedRequest: ow.optional.function,
         forefront: ow.optional.boolean,
         skipNavigation: ow.optional.boolean,
+        sessionId: ow.optional.string,
         limit: ow.optional.number,
         selector: ow.optional.string,
         baseUrl: ow.optional.string,
@@ -105,7 +107,9 @@ export async function enqueueLinks(options) {
         strategy: ow.optional.string.oneOf(Object.values(EnqueueStrategy)),
         waitForAllRequestsToBeAdded: ow.optional.boolean,
     }));
-    const { requestQueue, limit, urls, pseudoUrls, exclude, globs, regexps, transformRequestFunction, forefront, waitForAllRequestsToBeAdded, robotsTxtFile, onSkippedRequest, } = options;
+    const { requestManager, limit, urls, 
+    // oxlint-disable-next-line typescript/no-deprecated -- still accepted for backwards compat
+    pseudoUrls, exclude, globs, regexps, transformRequestFunction, forefront, waitForAllRequestsToBeAdded, robotsTxtFile, onSkippedRequest, } = options;
     const urlExcludePatternObjects = [];
     const urlPatternObjects = [];
     if (exclude?.length) {
@@ -119,7 +123,7 @@ export async function enqueueLinks(options) {
         }
     }
     if (pseudoUrls?.length) {
-        log.deprecated('`pseudoUrls` option is deprecated, use `globs` or `regexps` instead');
+        serviceLocator.getLogger().deprecated('`pseudoUrls` option is deprecated, use `globs` or `regexps` instead');
         urlPatternObjects.push(...constructRegExpObjectsFromPseudoUrls(pseudoUrls));
     }
     if (globs?.length) {
@@ -167,6 +171,16 @@ export async function enqueueLinks(options) {
                 break;
         }
     }
+    async function reportSkippedRequests(skippedRequests, reason) {
+        if (onSkippedRequest && skippedRequests.length > 0) {
+            await Promise.all(skippedRequests.map((request) => {
+                return onSkippedRequest({
+                    url: request.url,
+                    reason: request.skippedReason ?? reason,
+                });
+            }));
+        }
+    }
     let requestOptions = createRequestOptions(urls, options);
     if (robotsTxtFile) {
         const skippedRequests = [];
@@ -177,31 +191,38 @@ export async function enqueueLinks(options) {
             skippedRequests.push(request);
             return false;
         });
-        if (onSkippedRequest && skippedRequests.length > 0) {
-            await Promise.all(skippedRequests.map((request) => {
-                return onSkippedRequest({ url: request.url, reason: 'robotsTxt' });
-            }));
-        }
+        await reportSkippedRequests(skippedRequests, 'robotsTxt');
     }
-    if (transformRequestFunction) {
-        requestOptions = requestOptions
-            .map((request) => transformRequestFunction(request))
-            .filter((r) => !!r);
-    }
-    function createFilteredRequests() {
-        // No user provided patterns means we can skip an extra filtering step
+    async function createFilteredRequests() {
+        const skippedRequests = [];
+        // Step 1: Filter request options by exclude patterns, user patterns (globs/regexps), and strategy patterns.
+        // Pattern-level options (label, userData, method, etc.) are merged during this step.
+        let filteredOptions;
         if (urlPatternObjects.length === 0) {
-            return createRequests(requestOptions, enqueueStrategyPatterns, urlExcludePatternObjects, options.strategy);
+            filteredOptions = filterRequestOptionsByPatterns(requestOptions, enqueueStrategyPatterns.length > 0 ? enqueueStrategyPatterns : undefined, urlExcludePatternObjects, options.strategy, (url) => skippedRequests.push(url));
         }
-        // Generate requests based on the user patterns first
-        const generatedRequestsFromUserFilters = createRequests(requestOptions, urlPatternObjects, urlExcludePatternObjects, options.strategy);
-        // ...then filter them by the enqueue links strategy (making this an AND check)
-        return filterRequestsByPatterns(generatedRequestsFromUserFilters, enqueueStrategyPatterns);
+        else {
+            // Filter by user patterns first (with exclude)
+            const afterUserPatterns = filterRequestOptionsByPatterns(requestOptions, urlPatternObjects, urlExcludePatternObjects, options.strategy, (url) => skippedRequests.push(url));
+            // ...then filter by the enqueue links strategy (making this an AND check)
+            filteredOptions = filterRequestOptionsByPatterns(afterUserPatterns, enqueueStrategyPatterns.length > 0 ? enqueueStrategyPatterns : undefined, [], options.strategy, (url) => skippedRequests.push(url));
+        }
+        await reportSkippedRequests(skippedRequests.map((url) => ({ url })), 'filters');
+        // Step 2: Apply transformRequestFunction on request options - it has the highest priority
+        if (transformRequestFunction) {
+            const skippedByTransform = [];
+            filteredOptions = applyRequestTransform(filteredOptions, transformRequestFunction, (r) => skippedByTransform.push(r));
+            await reportSkippedRequests(skippedByTransform, 'transform');
+        }
+        // Step 3: Create Request instances from the final request options
+        return filteredOptions.map((opts) => new Request(opts));
     }
-    let requests = createFilteredRequests();
-    if (limit)
+    let requests = await createFilteredRequests();
+    if (typeof limit === 'number' && limit < requests.length) {
+        await reportSkippedRequests(requests.slice(limit), 'enqueueLimit');
         requests = requests.slice(0, limit);
-    const { addedRequests } = await requestQueue.addRequestsBatched(requests, {
+    }
+    const { addedRequests } = await requestManager.addRequestsBatched(requests, {
         forefront,
         waitForAllRequestsToBeAdded,
     });

package/enqueue_links/enqueue_links.js.map

@@ -1 +1 @@
-{"version":3,"file":"enqueue_links.js","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAGlC,OAAO,GAAG,MAAM,YAAY,CAAC;AAK7B,OAAO,EACH,6BAA6B,EAC7B,oCAAoC,EACpC,iCAAiC,EACjC,oBAAoB,EACpB,cAAc,EACd,wBAAwB,GAC3B,MAAM,aAAa,CAAC;AA+JrB;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,CAAN,IAAY,eAgCX;AAhCD,WAAY,eAAe;IACvB;;OAEG;IACH,8BAAW,CAAA;IAEX;;;;;;OAMG;IACH,iDAA8B,CAAA;IAE9B;;;;;;OAMG;IACH,6CAA0B,CAAA;IAE1B;;;;;;OAMG;IACH,6CAA0B,CAAA;AAC9B,CAAC,EAhCW,eAAe,KAAf,eAAe,QAgC1B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAC9B,OAAkE;IAElE,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,UAAU,CAChB;YACI,4JAA4J;YAC5J,kHAAkH;SACrH,CAAC,IAAI,CAAC,IAAI,CAAC,CACf,CAAC;IACN,CAAC;IAED,EAAE,CACE,OAAc,EACd,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QACjB,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;QAChC,YAAY,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,EAAE,YAAY,CAAC;QACjE,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC;QACtD,gBAAgB,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ;QACtC,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QAC9B,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QACnC,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC3B,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAClF,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAC7E,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAC7B,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CACvF;QACD,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;QACjF,wBAAwB,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ;QAC9C,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;QAClE,2BAA2B,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;KACnD,CAAC,CACL,CAAC;IAEF,MAAM,EACF,YAAY,EACZ,KAAK,EACL,IAAI,EACJ,UAAU,EACV,OAAO,EACP,KAAK,EACL,OAAO,EACP,wBAAwB,EACxB,SAAS,EACT,2BAA2B,EAC3B,aAAa,EACb,gBAAgB,GACnB,GAAG,OAAO,CAAC;IAEZ,MAAM,wBAAwB,GAAuB,EAAE,CAAC;IACxD,MAAM,iBAAiB,GAAuB,EAAE,CAAC;IAEjD,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YACzB,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;gBAC7C,wBAAwB,CAAC,IAAI,CAAC,GAAG,6BAA6B,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC5E,CAAC;iBAAM,IAAI,IAAI,YAAY,MAAM,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;gBACpD,wBAAwB,CAAC,IAAI,CAAC,GAAG,iCAAiC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAChF,CAAC;QACL,CAAC;IACL,CAAC;IAED,IAAI,UAAU,EAAE,MAAM,EAAE,CAAC;QACrB,GAAG,CAAC,UAAU,CAAC,qEAAqE,CAAC,CAAC;QACtF,iBAAiB,CAAC,IAAI,CAAC,GAAG,oCAAoC,CAAC,UAAU,CAAC,CAAC,CAAC;IAChF,CAAC;IAED,IAAI,KAAK,EAAE,MAAM,EAAE,CAAC;QAChB,iBAAiB,CAAC,IAAI,CAAC,GAAG,6BAA6B,CAAC,KAAK,CAAC,CAAC,CAAC;IACpE,CAAC;IAED,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,iBAAiB,CAAC,IAAI,CAAC,GAAG,iCAAiC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC1E,CAAC;IAED,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,CAAC;QAC5B,OAAO,CAAC,QAAQ,KAAK,eAAe,CAAC,YAAY,CAAC;IACtD,CAAC;IAED,MAAM,uBAAuB,GAAuB,EAAE,CAAC;IAEvD,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QAClB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAErC,QAAQ,OAAO,CAAC,QAAQ,EAAE,CAAC;YACvB,KAAK,eAAe,CAAC,YAAY;gBAC7B,sFAAsF;gBACtF,uFAAuF;gBACvF,yCAAyC;gBACzC,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC;gBAC7E,MAAM;YACV,KAAK,eAAe,CAAC,UAAU,CAAC,CAAC,CAAC;gBAC9B,4CAA4C;gBAC5C,MAAM,eAAe,GAAG,SAAS,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;gBAExE,IAAI,eAAe,EAAE,CAAC;oBAClB,gHAAgH;oBAChH,GAAG,CAAC,QAAQ,GAAG,eAAe,CAAC;oBAC/B,uBAAuB,CAAC,IAAI,CACxB,EAAE,IAAI,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,eAAe,EAAE,KAAK,eAAe,EAAE,CAAC,KAAK,CAAC,EAAE,EAC/F,EAAE,IAAI,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CACjD,CAAC;gBACN,CAAC;qBAAM,CAAC;oBACJ,6FAA6F;oBAC7F,4BAA4B;oBAC5B,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC;gBACjF,CAAC;gBAED,MAAM;YACV,CAAC;YACD,KAAK,eAAe,CAAC,UAAU,CAAC,CAAC,CAAC;gBAC9B,4EAA4E;gBAC5E,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC;gBAC3D,MAAM;YACV,CAAC;YACD,KAAK,eAAe,CAAC,GAAG,CAAC;YACzB;gBACI,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC;gBACxD,MAAM;QACd,CAAC;IACL,CAAC;IAED,IAAI,cAAc,GAAG,oBAAoB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAEzD,IAAI,aAAa,EAAE,CAAC;QAChB,MAAM,eAAe,GAAqB,EAAE,CAAC;QAE7C,cAAc,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE;YAC/C,IAAI,aAAa,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;gBACvC,OAAO,IAAI,CAAC;YAChB,CAAC;YAED,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC9B,OAAO,KAAK,CAAC;QACjB,CAAC,CAAC,CAAC;QAEH,IAAI,gBAAgB,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACjD,MAAM,OAAO,CAAC,GAAG,CACb,eAAe,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;gBAC5B,OAAO,gBAAgB,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;YACvE,CAAC,CAAC,CACL,CAAC;QACN,CAAC;IACL,CAAC;IAED,IAAI,wBAAwB,EAAE,CAAC;QAC3B,cAAc,GAAG,cAAc;aAC1B,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,wBAAwB,CAAC,OAAO,CAAC,CAAC;aACnD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAqB,CAAC;IAChD,CAAC;IAED,SAAS,sBAAsB;QAC3B,sEAAsE;QACtE,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,OAAO,cAAc,CAAC,cAAc,EAAE,uBAAuB,EAAE,wBAAwB,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC/G,CAAC;QAED,qDAAqD;QACrD,MAAM,gCAAgC,GAAG,cAAc,CACnD,cAAc,EACd,iBAAiB,EACjB,wBAAwB,EACxB,OAAO,CAAC,QAAQ,CACnB,CAAC;QACF,+EAA+E;QAC/E,OAAO,wBAAwB,CAAC,gCAAgC,EAAE,uBAAuB,CAAC,CAAC;IAC/F,CAAC;IAED,IAAI,QAAQ,GAAG,sBAAsB,EAAE,CAAC;IACxC,IAAI,KAAK;QAAE,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAE/C,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,YAAY,CAAC,kBAAkB,CAAC,QAAQ,EAAE;QACtE,SAAS;QACT,2BAA2B;KAC9B,CAAC,CAAC;IAEH,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,mBAAmB,EAAE,EAAE,EAAE,CAAC;AACzE,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,sCAAsC,CAAC,EACnD,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,mBAAmB,GACN;IACb,wCAAwC;IACxC,IAAI,mBAAmB,EAAE,CAAC;QACtB,OAAO,mBAAmB,CAAC;IAC/B,CAAC;IAED,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,kBAAkB,CAAC,CAAC,MAAM,CAAC;IAC7D,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,eAAe,IAAI,kBAAkB,CAAC,CAAC,MAAM,CAAC;IAE7E,6DAA6D;IAC7D,IAAI,eAAe,KAAK,eAAe,CAAC,GAAG,EAAE,CAAC;QAC1C,OAAO,cAAc,CAAC;IAC1B,CAAC;IAED,0HAA0H;IAC1H,+IAA+I;IAC/I,4DAA4D;IAC5D,IAAI,eAAe,KAAK,eAAe,CAAC,UAAU,EAAE,CAAC;QACjD,MAAM,gBAAgB,GAAG,SAAS,CAAC,iBAAiB,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAE,CAAC;QAC/E,MAAM,aAAa,GAAG,SAAS,CAAC,cAAc,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAE,CAAC;QAEzE,IAAI,gBAAgB,KAAK,aAAa,EAAE,CAAC;YACrC,OAAO,cAAc,CAAC;QAC1B,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,2JAA2J;IAC3J,mCAAmC;IACnC,OAAO,iBAAiB,CAAC;AAC7B,CAAC;AAYD;;GAEG;AACH,SAAS,gBAAgB,CAAC,OAAe;IACrC,OAAO,OAAO,CAAC,OAAO,CAAC,gBAAgB,EAAE,aAAa,CAAC,CAAC;AAC5D,CAAC"}
+{"version":3,"file":"enqueue_links.js","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAIlC,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACxC,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAgBvD,OAAO,EACH,qBAAqB,EACrB,6BAA6B,EAC7B,oCAAoC,EACpC,iCAAiC,EACjC,oBAAoB,EACpB,8BAA8B,GACjC,MAAM,aAAa,CAAC;AA2KrB;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,CAAN,IAAY,eAgCX;AAhCD,WAAY,eAAe;IACvB;;OAEG;IACH,8BAAW,CAAA;IAEX;;;;;;OAMG;IACH,iDAA8B,CAAA;IAE9B;;;;;;OAMG;IACH,6CAA0B,CAAA;IAE1B;;;;;;OAMG;IACH,6CAA0B,CAAA;AAC9B,CAAC,EAhCW,eAAe,KAAf,eAAe,QAgC1B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAC9B,OAOC;IAED,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,UAAU,CAChB;YACI,4JAA4J;YAC5J,kHAAkH;SACrH,CAAC,IAAI,CAAC,IAAI,CAAC,CACf,CAAC;IACN,CAAC;IAED,EAAE,CACE,OAAc,EACd,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QACjB,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;QAChC,cAAc,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,oBAAoB,CAAC;QACvD,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC;QACtD,gBAAgB,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ;QACtC,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QAC9B,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QACnC,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC7B,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC3B,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAClF,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAC7E,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAC7B,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CACvF;QACD,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;QACjF,wBAAwB,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ;QAC9C,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;QAClE,2BAA2B,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;KACnD,CAAC,CACL,CAAC;IAEF,MAAM,EACF,cAAc,EACd,KAAK,EACL,IAAI;IACJ,2FAA2F;IAC3F,UAAU,EACV,OAAO,EACP,KAAK,EACL,OAAO,EACP,wBAAwB,EACxB,SAAS,EACT,2BAA2B,EAC3B,aAAa,EACb,gBAAgB,GACnB,GAAG,OAAO,CAAC;IAEZ,MAAM,wBAAwB,GAAuB,EAAE,CAAC;IACxD,MAAM,iBAAiB,GAAuB,EAAE,CAAC;IAEjD,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YACzB,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;gBAC7C,wBAAwB,CAAC,IAAI,CAAC,GAAG,6BAA6B,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC5E,CAAC;iBAAM,IAAI,IAAI,YAAY,MAAM,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;gBACpD,wBAAwB,CAAC,IAAI,CAAC,GAAG,iCAAiC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAChF,CAAC;QACL,CAAC;IACL,CAAC;IAED,IAAI,UAAU,EAAE,MAAM,EAAE,CAAC;QACrB,cAAc,CAAC,SAAS,EAAE,CAAC,UAAU,CAAC,qEAAqE,CAAC,CAAC;QAC7G,iBAAiB,CAAC,IAAI,CAAC,GAAG,oCAAoC,CAAC,UAAU,CAAC,CAAC,CAAC;IAChF,CAAC;IAED,IAAI,KAAK,EAAE,MAAM,EAAE,CAAC;QAChB,iBAAiB,CAAC,IAAI,CAAC,GAAG,6BAA6B,CAAC,KAAK,CAAC,CAAC,CAAC;IACpE,CAAC;IAED,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,iBAAiB,CAAC,IAAI,CAAC,GAAG,iCAAiC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC1E,CAAC;IAED,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,CAAC;QAC5B,OAAO,CAAC,QAAQ,KAAK,eAAe,CAAC,YAAY,CAAC;IACtD,CAAC;IAED,MAAM,uBAAuB,GAAuB,EAAE,CAAC;IAEvD,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QAClB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAErC,QAAQ,OAAO,CAAC,QAAQ,EAAE,CAAC;YACvB,KAAK,eAAe,CAAC,YAAY;gBAC7B,sFAAsF;gBACtF,uFAAuF;gBACvF,yCAAyC;gBACzC,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC;gBAC7E,MAAM;YACV,KAAK,eAAe,CAAC,UAAU,CAAC,CAAC,CAAC;gBAC9B,4CAA4C;gBAC5C,MAAM,eAAe,GAAG,SAAS,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;gBAExE,IAAI,eAAe,EAAE,CAAC;oBAClB,gHAAgH;oBAChH,GAAG,CAAC,QAAQ,GAAG,eAAe,CAAC;oBAC/B,uBAAuB,CAAC,IAAI,CACxB,EAAE,IAAI,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,eAAe,EAAE,KAAK,eAAe,EAAE,CAAC,KAAK,CAAC,EAAE,EAC/F,EAAE,IAAI,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CACjD,CAAC;gBACN,CAAC;qBAAM,CAAC;oBACJ,6FAA6F;oBAC7F,4BAA4B;oBAC5B,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC;gBACjF,CAAC;gBAED,MAAM;YACV,CAAC;YACD,KAAK,eAAe,CAAC,UAAU,CAAC,CAAC,CAAC;gBAC9B,4EAA4E;gBAC5E,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC;gBAC3D,MAAM;YACV,CAAC;YACD,KAAK,eAAe,CAAC,GAAG,CAAC;YACzB;gBACI,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC;gBACxD,MAAM;QACd,CAAC;IACL,CAAC;IAED,KAAK,UAAU,qBAAqB,CAChC,eAAwE,EACxE,MAA4B;QAE5B,IAAI,gBAAgB,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACjD,MAAM,OAAO,CAAC,GAAG,CACb,eAAe,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;gBAC5B,OAAO,gBAAgB,CAAC;oBACpB,GAAG,EAAE,OAAO,CAAC,GAAG;oBAChB,MAAM,EAAE,OAAO,CAAC,aAAa,IAAI,MAAM;iBAC1C,CAAkB,CAAC;YACxB,CAAC,CAAC,CACL,CAAC;QACN,CAAC;IACL,CAAC;IAED,IAAI,cAAc,GAAG,oBAAoB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAEzD,IAAI,aAAa,EAAE,CAAC;QAChB,MAAM,eAAe,GAAqB,EAAE,CAAC;QAE7C,cAAc,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE;YAC/C,IAAI,aAAa,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;gBACvC,OAAO,IAAI,CAAC;YAChB,CAAC;YAED,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC9B,OAAO,KAAK,CAAC;QACjB,CAAC,CAAC,CAAC;QAEH,MAAM,qBAAqB,CAAC,eAAe,EAAE,WAAW,CAAC,CAAC;IAC9D,CAAC;IAED,KAAK,UAAU,sBAAsB;QACjC,MAAM,eAAe,GAAa,EAAE,CAAC;QAErC,4GAA4G;QAC5G,qFAAqF;QACrF,IAAI,eAAiC,CAAC;QACtC,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,eAAe,GAAG,8BAA8B,CAC5C,cAAc,EACd,uBAAuB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC,CAAC,SAAS,EACxE,wBAAwB,EACxB,OAAO,CAAC,QAAQ,EAChB,CAAC,GAAG,EAAE,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CACrC,CAAC;QACN,CAAC;aAAM,CAAC;YACJ,+CAA+C;YAC/C,MAAM,iBAAiB,GAAG,8BAA8B,CACpD,cAAc,EACd,iBAAiB,EACjB,wBAAwB,EACxB,OAAO,CAAC,QAAQ,EAChB,CAAC,GAAG,EAAE,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CACrC,CAAC;YACF,0EAA0E;YAC1E,eAAe,GAAG,8BAA8B,CAC5C,iBAAiB,EACjB,uBAAuB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC,CAAC,SAAS,EACxE,EAAE,EACF,OAAO,CAAC,QAAQ,EAChB,CAAC,GAAG,EAAE,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CACrC,CAAC;QACN,CAAC;QAED,MAAM,qBAAqB,CACvB,eAAe,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,EACvC,SAAS,CACZ,CAAC;QAEF,0FAA0F;QAC1F,IAAI,wBAAwB,EAAE,CAAC;YAC3B,MAAM,kBAAkB,GAAqB,EAAE,CAAC;YAChD,eAAe,GAAG,qBAAqB,CAAC,eAAe,EAAE,wBAAwB,EAAE,CAAC,CAAC,EAAE,EAAE,CACrF,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAC7B,CAAC;YACF,MAAM,qBAAqB,CAAC,kBAAkB,EAAE,WAAW,CAAC,CAAC;QACjE,CAAC;QAED,kEAAkE;QAClE,OAAO,eAAe,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;IAC5D,CAAC;IAED,IAAI,QAAQ,GAAG,MAAM,sBAAsB,EAAE,CAAC;IAE9C,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;QACvD,MAAM,qBAAqB,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,cAAc,CAAC,CAAC;QACnE,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IACxC,CAAC;IAED,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,cAAc,CAAC,kBAAkB,CAAC,QAAQ,EAAE;QACxE,SAAS;QACT,2BAA2B;KAC9B,CAAC,CAAC;IAEH,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,mBAAmB,EAAE,EAAE,EAAE,CAAC;AACzE,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,sCAAsC,CAAC,EACnD,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,mBAAmB,GACN;IACb,wCAAwC;IACxC,IAAI,mBAAmB,EAAE,CAAC;QACtB,OAAO,mBAAmB,CAAC;IAC/B,CAAC;IAED,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,kBAAkB,CAAC,CAAC,MAAM,CAAC;IAC7D,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,eAAe,IAAI,kBAAkB,CAAC,CAAC,MAAM,CAAC;IAE7E,6DAA6D;IAC7D,IAAI,eAAe,KAAK,eAAe,CAAC,GAAG,EAAE,CAAC;QAC1C,OAAO,cAAc,CAAC;IAC1B,CAAC;IAED,0HAA0H;IAC1H,+IAA+I;IAC/I,4DAA4D;IAC5D,IAAI,eAAe,KAAK,eAAe,CAAC,UAAU,EAAE,CAAC;QACjD,MAAM,gBAAgB,GAAG,SAAS,CAAC,iBAAiB,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAE,CAAC;QAC/E,MAAM,aAAa,GAAG,SAAS,CAAC,cAAc,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAE,CAAC;QAEzE,IAAI,gBAAgB,KAAK,aAAa,EAAE,CAAC;YACrC,OAAO,cAAc,CAAC;QAC1B,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,2JAA2J;IAC3J,mCAAmC;IACnC,OAAO,iBAAiB,CAAC;AAC7B,CAAC;AAYD;;GAEG;AACH,SAAS,gBAAgB,CAAC,OAAe;IACrC,OAAO,OAAO,CAAC,OAAO,CAAC,gBAAgB,EAAE,aAAa,CAAC,CAAC;AAC5D,CAAC"}

package/enqueue_links/shared.d.ts

@@ -1,5 +1,5 @@
+import type { Awaitable } from '@crawlee/types';
 import type { RequestOptions } from '../request.js';
-import { Request } from '../request.js';
 import type { EnqueueLinksOptions } from './enqueue_links.js';
 export { tryAbsoluteURL } from '@crawlee/utils';
 export type UrlPatternObject = {
@@ -18,6 +18,11 @@ export type RegExpObject = {
     regexp: RegExp;
 } & Pick<RequestOptions, 'method' | 'payload' | 'label' | 'userData' | 'headers'>;
 export type RegExpInput = RegExp | RegExpObject;
+export type SkippedRequestReason = 'robotsTxt' | 'limit' | 'enqueueLimit' | 'filters' | 'transform' | 'redirect' | 'depth';
+export type SkippedRequestCallback = (args: {
+    url: string;
+    reason: SkippedRequestReason;
+}) => Awaitable<void>;
 /**
  * @ignore
  */
@@ -45,23 +50,35 @@ export declare function validateGlobPattern(glob: string): string;
  */
 export declare function constructRegExpObjectsFromRegExps(regexps: readonly RegExpInput[]): RegExpObject[];
 /**
+ * Filters request options by URL patterns and merges pattern-level options (label, userData, method, payload, headers)
+ * from the first matching pattern into each RequestOptions entry.
+ *
+ * When `includePatterns` is empty/undefined, all options pass through (only exclude filtering applies).
  * @ignore
  */
-export declare function createRequests(requestOptions: (string | RequestOptions)[], urlPatternObjects?: UrlPatternObject[], excludePatternObjects?: UrlPatternObject[], strategy?: EnqueueLinksOptions['strategy']): Request[];
-export declare function filterRequestsByPatterns(requests: Request[], patterns?: UrlPatternObject[]): Request[];
+export declare function filterRequestOptionsByPatterns(requestOptions: RequestOptions[], includePatterns: UrlPatternObject[] | undefined, excludePatterns?: UrlPatternObject[], strategy?: EnqueueLinksOptions['strategy'], onSkippedUrl?: (url: string) => void): RequestOptions[];
 /**
  * @ignore
  */
-export declare function createRequestOptions(sources: readonly (string | Record<string, unknown>)[], options?: Pick<EnqueueLinksOptions, 'label' | 'userData' | 'baseUrl' | 'skipNavigation' | 'strategy'>): RequestOptions[];
+export declare function createRequestOptions(sources: readonly (string | Record<string, unknown>)[], options?: Pick<EnqueueLinksOptions, 'label' | 'userData' | 'baseUrl' | 'skipNavigation' | 'sessionId' | 'strategy'>): RequestOptions[];
 /**
- * Takes an Apify {@link RequestOptions} object and changes its attributes in a desired way. This user-function is used
- * {@link enqueueLinks} to modify requests before enqueuing them.
+ * Takes a {@link RequestOptions} object and changes its attributes in a desired way. This user-function is used
+ * by {@link enqueueLinks} to modify request options before they are converted to {@link Request} instances.
  */
 export interface RequestTransform {
     /**
      * @param original Request options to be modified.
-     * @returns The modified request options to enqueue.
+     * @returns The modified request options to enqueue, `'unchanged'` to keep the original options as-is,
+     *   or a falsy value / `'skip'` to exclude the request from the queue.
      */
-    (original: RequestOptions): RequestOptions | false | undefined | null;
+    (original: RequestOptions): RequestOptions | false | undefined | null | 'skip' | 'unchanged';
 }
+/**
+ * Applies a {@link RequestTransform} function to a list of request options.
+ * Options for which the transform returns a falsy value are removed from the list.
+ * @param onSkipped Called with the original request options when the transform returns a falsy value (i.e. the request is skipped).
+ * @ignore
+ * @internal
+ */
+export declare function applyRequestTransform(requestOptions: RequestOptions[], transformFn: RequestTransform, onSkipped?: (requestOptions: RequestOptions) => void): RequestOptions[];
 //# sourceMappingURL=shared.d.ts.map

package/enqueue_links/shared.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/shared.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACxC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAE9D,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAYhD,MAAM,MAAM,gBAAgB,GAAG;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB,GAAG,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAAC,CAAC;AAElF,MAAM,MAAM,eAAe,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CACjD,cAAc,EACd,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAC1D,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG,MAAM,GAAG,eAAe,CAAC;AAEtD,MAAM,MAAM,UAAU,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAC5C,cAAc,EACd,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAC1D,CAAC;AAEF,MAAM,MAAM,SAAS,GAAG,MAAM,GAAG,UAAU,CAAC;AAE5C,MAAM,MAAM,YAAY,GAAG;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAChD,cAAc,EACd,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAC1D,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,YAAY,CAAC;AAEhD;;GAEG;AACH,wBAAgB,8BAA8B,CAC1C,IAAI,EAAE,SAAS,GAAG,WAAW,GAAG,cAAc,EAC9C,OAAO,EAAE,YAAY,GAAG,UAAU,GACnC,IAAI,CAMN;AAED;;;;GAIG;AACH,wBAAgB,oCAAoC,CAAC,UAAU,EAAE,SAAS,cAAc,EAAE,GAAG,YAAY,EAAE,CAiB1G;AAED;;;;GAIG;AACH,wBAAgB,6BAA6B,CAAC,KAAK,EAAE,SAAS,SAAS,EAAE,GAAG,UAAU,EAAE,CAkCvF;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAKxD;AAED;;;;GAIG;AACH,wBAAgB,iCAAiC,CAAC,OAAO,EAAE,SAAS,WAAW,EAAE,GAAG,YAAY,EAAE,CAgBjG;AAED;;GAEG;AACH,wBAAgB,cAAc,CAC1B,cAAc,EAAE,CAAC,MAAM,GAAG,cAAc,CAAC,EAAE,EAC3C,iBAAiB,CAAC,EAAE,gBAAgB,EAAE,EACtC,qBAAqB,GAAE,gBAAgB,EAAO,EAC9C,QAAQ,CAAC,EAAE,mBAAmB,CAAC,UAAU,CAAC,GAC3C,OAAO,EAAE,CA8BX;AAED,wBAAgB,wBAAwB,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,QAAQ,CAAC,EAAE,gBAAgB,EAAE,GAAG,OAAO,EAAE,CAoBtG;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAChC,OAAO,EAAE,SAAS,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,EAAE,EACtD,OAAO,GAAE,IAAI,CAAC,mBAAmB,EAAE,OAAO,GAAG,UAAU,GAAG,SAAS,GAAG,gBAAgB,GAAG,UAAU,CAAM,GAC1G,cAAc,EAAE,CA+BlB;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC7B;;;OAGG;IACH,CAAC,QAAQ,EAAE,cAAc,GAAG,cAAc,GAAG,KAAK,GAAG,SAAS,GAAG,IAAI,CAAC;CACzE"}
+{"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/shared.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAKhD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAE9D,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAYhD,MAAM,MAAM,gBAAgB,GAAG;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB,GAAG,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAAC,CAAC;AAElF,MAAM,MAAM,eAAe,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CACjD,cAAc,EACd,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAC1D,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG,MAAM,GAAG,eAAe,CAAC;AAEtD,MAAM,MAAM,UAAU,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAC5C,cAAc,EACd,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAC1D,CAAC;AAEF,MAAM,MAAM,SAAS,GAAG,MAAM,GAAG,UAAU,CAAC;AAE5C,MAAM,MAAM,YAAY,GAAG;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAChD,cAAc,EACd,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAC1D,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,YAAY,CAAC;AAEhD,MAAM,MAAM,oBAAoB,GAC1B,WAAW,GACX,OAAO,GACP,cAAc,GACd,SAAS,GACT,WAAW,GACX,UAAU,GACV,OAAO,CAAC;AAEd,MAAM,MAAM,sBAAsB,GAAG,CAAC,IAAI,EAAE;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,oBAAoB,CAAA;CAAE,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAE9G;;GAEG;AACH,wBAAgB,8BAA8B,CAC1C,IAAI,EAAE,SAAS,GAAG,WAAW,GAAG,cAAc,EAC9C,OAAO,EAAE,YAAY,GAAG,UAAU,GACnC,IAAI,CAMN;AAED;;;;GAIG;AACH,wBAAgB,oCAAoC,CAAC,UAAU,EAAE,SAAS,cAAc,EAAE,GAAG,YAAY,EAAE,CAiB1G;AAED;;;;GAIG;AACH,wBAAgB,6BAA6B,CAAC,KAAK,EAAE,SAAS,SAAS,EAAE,GAAG,UAAU,EAAE,CAkCvF;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAKxD;AAED;;;;GAIG;AACH,wBAAgB,iCAAiC,CAAC,OAAO,EAAE,SAAS,WAAW,EAAE,GAAG,YAAY,EAAE,CAgBjG;AAED;;;;;;GAMG;AACH,wBAAgB,8BAA8B,CAC1C,cAAc,EAAE,cAAc,EAAE,EAChC,eAAe,EAAE,gBAAgB,EAAE,GAAG,SAAS,EAC/C,eAAe,GAAE,gBAAgB,EAAO,EACxC,QAAQ,CAAC,EAAE,mBAAmB,CAAC,UAAU,CAAC,EAC1C,YAAY,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,GACrC,cAAc,EAAE,CA4BlB;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAChC,OAAO,EAAE,SAAS,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,EAAE,EACtD,OAAO,GAAE,IAAI,CACT,mBAAmB,EACnB,OAAO,GAAG,UAAU,GAAG,SAAS,GAAG,gBAAgB,GAAG,WAAW,GAAG,UAAU,CAC5E,GACP,cAAc,EAAE,CAmClB;AAmBD;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC7B;;;;OAIG;IACH,CAAC,QAAQ,EAAE,cAAc,GAAG,cAAc,GAAG,KAAK,GAAG,SAAS,GAAG,IAAI,GAAG,MAAM,GAAG,WAAW,CAAC;CAChG;AAED;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CACjC,cAAc,EAAE,cAAc,EAAE,EAChC,WAAW,EAAE,gBAAgB,EAC7B,SAAS,CAAC,EAAE,CAAC,cAAc,EAAE,cAAc,KAAK,IAAI,GACrD,cAAc,EAAE,CAkBlB"}

package/enqueue_links/shared.js

@@ -1,7 +1,6 @@
 import { URL } from 'node:url';
-import { minimatch } from 'minimatch';
+import { Minimatch } from 'minimatch';
 import { purlToRegExp } from '@apify/pseudo_url';
-import { Request } from '../request.js';
 export { tryAbsoluteURL } from '@crawlee/utils';
 const MAX_ENQUEUE_LINKS_CACHE_SIZE = 1000;
 /**
@@ -110,51 +109,37 @@ export function constructRegExpObjectsFromRegExps(regexps) {
     });
 }
 /**
+ * Filters request options by URL patterns and merges pattern-level options (label, userData, method, payload, headers)
+ * from the first matching pattern into each RequestOptions entry.
+ *
+ * When `includePatterns` is empty/undefined, all options pass through (only exclude filtering applies).
  * @ignore
  */
-export function createRequests(requestOptions, urlPatternObjects, excludePatternObjects = [], strategy) {
+export function filterRequestOptionsByPatterns(requestOptions, includePatterns, excludePatterns = [], strategy, onSkippedUrl) {
+    const excludeMatchers = excludePatterns.map(createPatternObjectMatcher);
+    const includeMatchers = includePatterns?.length ? includePatterns.map(createPatternObjectMatcher) : undefined;
     return requestOptions
-        .map((opts) => ({ url: typeof opts === 'string' ? opts : opts.url, opts }))
         .filter(({ url }) => {
-        return !excludePatternObjects.some((excludePatternObject) => {
-            const { regexp, glob } = excludePatternObject;
-            return (regexp && url.match(regexp)) || (glob && minimatch(url, glob, { nocase: true }));
-        });
+        const matchesExclude = excludeMatchers.some(({ match }) => match(url));
+        if (matchesExclude) {
+            onSkippedUrl?.(url);
+        }
+        return !matchesExclude;
     })
-        .map(({ url, opts }) => {
-        if (!urlPatternObjects || !urlPatternObjects.length) {
-            return new Request(typeof opts === 'string' ? { url: opts, enqueueStrategy: strategy } : { ...opts });
-        }
-        for (const urlPatternObject of urlPatternObjects) {
-            const { regexp, glob, ...requestRegExpOptions } = urlPatternObject;
-            if ((regexp && url.match(regexp)) || (glob && minimatch(url, glob, { nocase: true }))) {
-                const request = typeof opts === 'string'
-                    ? { url: opts, ...requestRegExpOptions, enqueueStrategy: strategy }
-                    : { ...opts, ...requestRegExpOptions, enqueueStrategy: strategy };
-                return new Request(request);
+        .map((opts) => {
+        if (!includeMatchers) {
+            return { ...opts, enqueueStrategy: strategy };
+        }
+        for (const { match, glob, regexp, ...patternOptions } of includeMatchers) {
+            if (match(opts.url)) {
+                return { ...opts, ...patternOptions, enqueueStrategy: strategy };
             }
         }
         // didn't match any positive pattern
+        onSkippedUrl?.(opts.url);
         return null;
     })
-        .filter((request) => request);
-}
-export function filterRequestsByPatterns(requests, patterns) {
-    if (!patterns?.length) {
-        return requests;
-    }
-    const filtered = [];
-    for (const request of requests) {
-        for (const urlPatternObject of patterns) {
-            const { regexp, glob } = urlPatternObject;
-            if ((regexp && request.url.match(regexp)) || (glob && minimatch(request.url, glob, { nocase: true }))) {
-                filtered.push(request);
-                // Break the pattern loop, as we already matched this request once
-                break;
-            }
-        }
-    }
-    return filtered;
+        .filter((opts) => opts !== null);
 }
 /**
  * @ignore
@@ -184,7 +169,54 @@ export function createRequestOptions(sources, options = {}) {
         if (options.skipNavigation) {
             requestOptions.skipNavigation = true;
         }
+        if (options.sessionId) {
+            requestOptions.sessionId = options.sessionId;
+        }
         return requestOptions;
     });
 }
+/**
+ * @ignore
+ */
+function createPatternObjectMatcher(urlPatternObject) {
+    const { regexp, glob } = urlPatternObject;
+    let match;
+    if (regexp) {
+        match = (url) => regexp.test(url);
+    }
+    else if (glob) {
+        const m = new Minimatch(glob, { nocase: true });
+        match = (url) => m.match(url);
+    }
+    else {
+        match = () => false;
+    }
+    return { ...urlPatternObject, match };
+}
+/**
+ * Applies a {@link RequestTransform} function to a list of request options.
+ * Options for which the transform returns a falsy value are removed from the list.
+ * @param onSkipped Called with the original request options when the transform returns a falsy value (i.e. the request is skipped).
+ * @ignore
+ * @internal
+ */
+export function applyRequestTransform(requestOptions, transformFn, onSkipped) {
+    return requestOptions
+        .map((opts) => {
+        const transformed = transformFn(opts);
+        if (transformed === 'skip') {
+            onSkipped?.(opts);
+            return null;
+        }
+        if (transformed === 'unchanged') {
+            return opts;
+        }
+        if (!transformed) {
+            onSkipped?.(opts);
+            return null;
+        }
+        return transformed;
+    })
+        .filter((r) => r !== null);
+}
 //# sourceMappingURL=shared.js.map

package/enqueue_links/shared.js.map

@@ -1 +1 @@
-{"version":3,"file":"shared.js","sourceRoot":"","sources":["../../src/enqueue_links/shared.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAE/B,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEtC,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAGjD,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAGxC,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAEhD,MAAM,4BAA4B,GAAG,IAAI,CAAC;AAE1C;;;;;GAKG;AACH,MAAM,wBAAwB,GAAG,IAAI,GAAG,EAAE,CAAC;AA4B3C;;GAEG;AACH,MAAM,UAAU,8BAA8B,CAC1C,IAA8C,EAC9C,OAAkC;IAElC,wBAAwB,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC5C,IAAI,wBAAwB,CAAC,IAAI,GAAG,4BAA4B,EAAE,CAAC;QAC/D,MAAM,GAAG,GAAG,wBAAwB,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;QACzD,wBAAwB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACzC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,oCAAoC,CAAC,UAAqC;IACtF,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC3B,mCAAmC;QACnC,IAAI,YAAY,GAAG,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtD,IAAI,YAAY;YAAE,OAAO,YAAY,CAAC;QAEtC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,YAAY,GAAG,EAAE,MAAM,EAAE,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC;QAClD,CAAC;aAAM,CAAC;YACJ,MAAM,EAAE,IAAI,EAAE,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC;YACzC,YAAY,GAAG,EAAE,MAAM,EAAE,YAAY,CAAC,IAAI,CAAC,EAAE,GAAG,cAAc,EAAE,CAAC;QACrE,CAAC;QAED,8BAA8B,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QAEnD,OAAO,YAAY,CAAC;IACxB,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,6BAA6B,CAAC,KAA2B;IACrE,OAAO,KAAK;SACP,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;QACb,uCAAuC;QACvC,IAAI,CAAC,IAAI,EAAE,CAAC;YACR,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;QAClC,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACZ,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;QACvC,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACV,8BAA8B;QAC9B,IAAI,UAAU,GAAG,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpD,IAAI,UAAU;YAAE,OAAO,UAAU,CAAC;QAElC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,UAAU,GAAG,EAAE,IAAI,EAAE,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC;QACrD,CAAC;aAAM,CAAC;YACJ,MAAM,EAAE,IAAI,EAAE,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC;YACzC,UAAU,GAAG,EAAE,IAAI,EAAE,mBAAmB,CAAC,IAAI,CAAC,EAAE,GAAG,cAAc,EAAE,CAAC;QACxE,CAAC;QAED,8BAA8B,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QAEjD,OAAO,UAAU,CAAC;IACtB,CAAC,CAAC,CAAC;AACX,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAAY;IAC5C,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAChC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,8BAA8B,WAAW,mCAAmC,CAAC,CAAC;IAClG,OAAO,WAAW,CAAC;AACvB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iCAAiC,CAAC,OAA+B;IAC7E,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,gCAAgC;QAChC,IAAI,YAAY,GAAG,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtD,IAAI,YAAY;YAAE,OAAO,YAAY,CAAC;QAEtC,IAAI,IAAI,YAAY,MAAM,EAAE,CAAC;YACzB,YAAY,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QACpC,CAAC;aAAM,CAAC;YACJ,YAAY,GAAG,IAAI,CAAC;QACxB,CAAC;QAED,8BAA8B,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QAEnD,OAAO,YAAY,CAAC;IACxB,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAC1B,cAA2C,EAC3C,iBAAsC,EACtC,wBAA4C,EAAE,EAC9C,QAA0C;IAE1C,OAAO,cAAc;SAChB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;SAC1E,MAAM,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;QAChB,OAAO,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC,oBAAoB,EAAE,EAAE;YACxD,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,oBAAoB,CAAC;YAC9C,OAAO,CAAC,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QAC7F,CAAC,CAAC,CAAC;IACP,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE;QACnB,IAAI,CAAC,iBAAiB,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,CAAC;YAClD,OAAO,IAAI,OAAO,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,eAAe,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC;QAC1G,CAAC;QAED,KAAK,MAAM,gBAAgB,IAAI,iBAAiB,EAAE,CAAC;YAC/C,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,oBAAoB,EAAE,GAAG,gBAAgB,CAAC;YACnE,IAAI,CAAC,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;gBACpF,MAAM,OAAO,GACT,OAAO,IAAI,KAAK,QAAQ;oBACpB,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,oBAAoB,EAAE,eAAe,EAAE,QAAQ,EAAE;oBACnE,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,GAAG,oBAAoB,EAAE,eAAe,EAAE,QAAQ,EAAE,CAAC;gBAE1E,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;YAChC,CAAC;QACL,CAAC;QAED,oCAAoC;QACpC,OAAO,IAAI,CAAC;IAChB,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAc,CAAC;AACnD,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,QAAmB,EAAE,QAA6B;IACvF,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;QACpB,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,MAAM,QAAQ,GAAc,EAAE,CAAC;IAE/B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC7B,KAAK,MAAM,gBAAgB,IAAI,QAAQ,EAAE,CAAC;YACtC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,gBAAgB,CAAC;YAE1C,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,SAAS,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;gBACpG,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACvB,kEAAkE;gBAClE,MAAM;YACV,CAAC;QACL,CAAC;IACL,CAAC;IAED,OAAO,QAAQ,CAAC;AACpB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAChC,OAAsD,EACtD,UAAuG,EAAE;IAEzG,OAAO,OAAO;SACT,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CACT,OAAO,GAAG,KAAK,QAAQ;QACnB,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,eAAe,EAAE,OAAO,CAAC,QAAQ,EAAE;QACjD,CAAC,CAAE,EAAE,GAAG,GAAG,EAAE,eAAe,EAAE,OAAO,CAAC,QAAQ,EAAqB,CAC1E;SACA,MAAM,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;QAChB,IAAI,CAAC;YACD,OAAO,IAAI,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC;QAC9C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,cAAc,EAAE,EAAE;QACpB,cAAc,CAAC,GAAG,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,GAAG,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC;QACvE,cAAc,CAAC,QAAQ,KAAK,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;QAEnD,IAAI,OAAO,OAAO,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;YACpC,cAAc,CAAC,QAAQ,GAAG;gBACtB,GAAG,cAAc,CAAC,QAAQ;gBAC1B,KAAK,EAAE,OAAO,CAAC,KAAK;aACvB,CAAC;QACN,CAAC;QAED,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YACzB,cAAc,CAAC,cAAc,GAAG,IAAI,CAAC;QACzC,CAAC;QAED,OAAO,cAAc,CAAC;IAC1B,CAAC,CAAC,CAAC;AACX,CAAC"}
+{"version":3,"file":"shared.js","sourceRoot":"","sources":["../../src/enqueue_links/shared.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAG/B,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEtC,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAKjD,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAEhD,MAAM,4BAA4B,GAAG,IAAI,CAAC;AAE1C;;;;;GAKG;AACH,MAAM,wBAAwB,GAAG,IAAI,GAAG,EAAE,CAAC;AAuC3C;;GAEG;AACH,MAAM,UAAU,8BAA8B,CAC1C,IAA8C,EAC9C,OAAkC;IAElC,wBAAwB,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC5C,IAAI,wBAAwB,CAAC,IAAI,GAAG,4BAA4B,EAAE,CAAC;QAC/D,MAAM,GAAG,GAAG,wBAAwB,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;QACzD,wBAAwB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACzC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,oCAAoC,CAAC,UAAqC;IACtF,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC3B,mCAAmC;QACnC,IAAI,YAAY,GAAG,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtD,IAAI,YAAY;YAAE,OAAO,YAAY,CAAC;QAEtC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,YAAY,GAAG,EAAE,MAAM,EAAE,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC;QAClD,CAAC;aAAM,CAAC;YACJ,MAAM,EAAE,IAAI,EAAE,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC;YACzC,YAAY,GAAG,EAAE,MAAM,EAAE,YAAY,CAAC,IAAI,CAAC,EAAE,GAAG,cAAc,EAAE,CAAC;QACrE,CAAC;QAED,8BAA8B,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QAEnD,OAAO,YAAY,CAAC;IACxB,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,6BAA6B,CAAC,KAA2B;IACrE,OAAO,KAAK;SACP,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;QACb,uCAAuC;QACvC,IAAI,CAAC,IAAI,EAAE,CAAC;YACR,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;QAClC,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACZ,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;QACvC,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACV,8BAA8B;QAC9B,IAAI,UAAU,GAAG,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpD,IAAI,UAAU;YAAE,OAAO,UAAU,CAAC;QAElC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,UAAU,GAAG,EAAE,IAAI,EAAE,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC;QACrD,CAAC;aAAM,CAAC;YACJ,MAAM,EAAE,IAAI,EAAE,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC;YACzC,UAAU,GAAG,EAAE,IAAI,EAAE,mBAAmB,CAAC,IAAI,CAAC,EAAE,GAAG,cAAc,EAAE,CAAC;QACxE,CAAC;QAED,8BAA8B,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QAEjD,OAAO,UAAU,CAAC;IACtB,CAAC,CAAC,CAAC;AACX,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAAY;IAC5C,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAChC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,8BAA8B,WAAW,mCAAmC,CAAC,CAAC;IAClG,OAAO,WAAW,CAAC;AACvB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iCAAiC,CAAC,OAA+B;IAC7E,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,gCAAgC;QAChC,IAAI,YAAY,GAAG,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtD,IAAI,YAAY;YAAE,OAAO,YAAY,CAAC;QAEtC,IAAI,IAAI,YAAY,MAAM,EAAE,CAAC;YACzB,YAAY,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QACpC,CAAC;aAAM,CAAC;YACJ,YAAY,GAAG,IAAI,CAAC;QACxB,CAAC;QAED,8BAA8B,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QAEnD,OAAO,YAAY,CAAC;IACxB,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,8BAA8B,CAC1C,cAAgC,EAChC,eAA+C,EAC/C,kBAAsC,EAAE,EACxC,QAA0C,EAC1C,YAAoC;IAEpC,MAAM,eAAe,GAAG,eAAe,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;IACxE,MAAM,eAAe,GAAG,eAAe,EAAE,MAAM,CAAC,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAE9G,OAAO,cAAc;SAChB,MAAM,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;QAChB,MAAM,cAAc,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;QACvE,IAAI,cAAc,EAAE,CAAC;YACjB,YAAY,EAAE,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;QACD,OAAO,CAAC,cAAc,CAAC;IAC3B,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACV,IAAI,CAAC,eAAe,EAAE,CAAC;YACnB,OAAO,EAAE,GAAG,IAAI,EAAE,eAAe,EAAE,QAAQ,EAAE,CAAC;QAClD,CAAC;QAED,KAAK,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,cAAc,EAAE,IAAI,eAAe,EAAE,CAAC;YACvE,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBAClB,OAAO,EAAE,GAAG,IAAI,EAAE,GAAG,cAAc,EAAE,eAAe,EAAE,QAAQ,EAAE,CAAC;YACrE,CAAC;QACL,CAAC;QAED,oCAAoC;QACpC,YAAY,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACzB,OAAO,IAAI,CAAC;IAChB,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;AACzC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAChC,OAAsD,EACtD,UAGI,EAAE;IAEN,OAAO,OAAO;SACT,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CACT,OAAO,GAAG,KAAK,QAAQ;QACnB,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,eAAe,EAAE,OAAO,CAAC,QAAQ,EAAE;QACjD,CAAC,CAAE,EAAE,GAAG,GAAG,EAAE,eAAe,EAAE,OAAO,CAAC,QAAQ,EAAqB,CAC1E;SACA,MAAM,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;QAChB,IAAI,CAAC;YACD,OAAO,IAAI,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC;QAC9C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,cAAc,EAAE,EAAE;QACpB,cAAc,CAAC,GAAG,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,GAAG,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC;QACvE,cAAc,CAAC,QAAQ,KAAK,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;QAEnD,IAAI,OAAO,OAAO,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;YACpC,cAAc,CAAC,QAAQ,GAAG;gBACtB,GAAG,cAAc,CAAC,QAAQ;gBAC1B,KAAK,EAAE,OAAO,CAAC,KAAK;aACvB,CAAC;QACN,CAAC;QAED,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YACzB,cAAc,CAAC,cAAc,GAAG,IAAI,CAAC;QACzC,CAAC;QAED,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACpB,cAAc,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACjD,CAAC;QAED,OAAO,cAAc,CAAC;IAC1B,CAAC,CAAC,CAAC;AACX,CAAC;AAED;;GAEG;AACH,SAAS,0BAA0B,CAAC,gBAAkC;IAClE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,gBAAgB,CAAC;IAC1C,IAAI,KAAK,CAAC;IACV,IAAI,MAAM,EAAE,CAAC;QACT,KAAK,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC9C,CAAC;SAAM,IAAI,IAAI,EAAE,CAAC;QACd,MAAM,CAAC,GAAG,IAAI,SAAS,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;QAChD,KAAK,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC1C,CAAC;SAAM,CAAC;QACJ,KAAK,GAAG,GAAG,EAAE,CAAC,KAAK,CAAC;IACxB,CAAC;IACD,OAAO,EAAE,GAAG,gBAAgB,EAAE,KAAK,EAAE,CAAC;AAC1C,CAAC;AAeD;;;;;;GAMG;AACH,MAAM,UAAU,qBAAqB,CACjC,cAAgC,EAChC,WAA6B,EAC7B,SAAoD;IAEpD,OAAO,cAAc;SAChB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACV,MAAM,WAAW,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QACtC,IAAI,WAAW,KAAK,MAAM,EAAE,CAAC;YACzB,SAAS,EAAE,CAAC,IAAI,CAAC,CAAC;YAClB,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,IAAI,WAAW,KAAK,WAAW,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,IAAI,CAAC,WAAW,EAAE,CAAC;YACf,SAAS,EAAE,CAAC,IAAI,CAAC,CAAC;YAClB,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,OAAO,WAAW,CAAC;IACvB,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,CAAC,EAAuB,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;AACxD,CAAC"}

package/errors.d.ts

@@ -23,11 +23,41 @@ export declare class RetryRequestError extends Error {
     constructor(message?: string);
 }
 /**
- * Errors of `SessionError` type will trigger a session rotation.
+ * Errors of `SessionError` type retire the session associated with the request and trigger a regular retry.
  *
- * This error doesn't respect the `maxRequestRetries` option and has a separate limit of `maxSessionRotations`.
+ * The retry counts towards the `maxRequestRetries` limit, just like any other error.
  */
-export declare class SessionError extends RetryRequestError {
+export declare class SessionError extends Error {
     constructor(message?: string);
 }
+/**
+ * Thrown when a requested session is not found in the referenced SessionPool.
+ */
+export declare class MissingSessionError extends Error {
+    constructor(sessionId?: string);
+}
+export declare class ContextPipelineInterruptedError extends Error {
+    constructor(message?: string);
+}
+export declare class ContextPipelineInitializationError extends Error {
+    constructor(error: unknown, options?: ErrorOptions);
+}
+export declare class ContextPipelineCleanupError extends CriticalError {
+    constructor(error: unknown, options?: ErrorOptions);
+}
+export declare class RequestHandlerError extends Error {
+    constructor(error: unknown, options?: ErrorOptions);
+}
+/**
+ * Thrown when attempting to set a different service instance after one has already been retrieved.
+ */
+export declare class ServiceConflictError extends Error {
+    constructor(serviceName: string, newValue: unknown, existingValue: unknown);
+}
+/**
+ * Thrown by crawlers when `skipNavigation` is used on a request.
+ * Subclasses can catch this error to skip their own navigation-dependent logic.
+ */
+export declare class NavigationSkippedError extends NonRetryableError {
+}
 //# sourceMappingURL=errors.d.ts.map

package/errors.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,KAAK;CAAG;AAE/C;;;GAGG;AACH,qBAAa,aAAc,SAAQ,iBAAiB;CAAG;AAEvD;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,aAAa;CAAG;AAEvD;;;;GAIG;AACH,qBAAa,iBAAkB,SAAQ,KAAK;gBAC5B,OAAO,CAAC,EAAE,MAAM;CAG/B;AAED;;;;GAIG;AACH,qBAAa,YAAa,SAAQ,iBAAiB;gBACnC,OAAO,CAAC,EAAE,MAAM;CAG/B"}
+{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAEA;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,KAAK;CAAG;AAE/C;;;GAGG;AACH,qBAAa,aAAc,SAAQ,iBAAiB;CAAG;AAEvD;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,aAAa;CAAG;AAEvD;;;;GAIG;AACH,qBAAa,iBAAkB,SAAQ,KAAK;gBAC5B,OAAO,CAAC,EAAE,MAAM;CAG/B;AAED;;;;GAIG;AACH,qBAAa,YAAa,SAAQ,KAAK;gBACvB,OAAO,CAAC,EAAE,MAAM;CAG/B;AAED;;GAEG;AACH,qBAAa,mBAAoB,SAAQ,KAAK;gBAC9B,SAAS,CAAC,EAAE,MAAM;CAKjC;AAED,qBAAa,+BAAgC,SAAQ,KAAK;gBAC1C,OAAO,CAAC,EAAE,MAAM;CAG/B;AAED,qBAAa,kCAAmC,SAAQ,KAAK;gBAC7C,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,YAAY;CAGrD;AAED,qBAAa,2BAA4B,SAAQ,aAAa;gBAC9C,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,YAAY;CAGrD;AAED,qBAAa,mBAAoB,SAAQ,KAAK;gBAC9B,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,YAAY;CAGrD;AAED;;GAEG;AACH,qBAAa,oBAAqB,SAAQ,KAAK;gBAC/B,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,EAAE,OAAO;CAM7E;AAED;;;GAGG;AACH,qBAAa,sBAAuB,SAAQ,iBAAiB;CAAG"}

package/errors.js

@@ -1,3 +1,4 @@
+import { inspectValue } from '@crawlee/utils';
 /**
  * Errors of `NonRetryableError` type will never be retried by the crawler.
  */
@@ -25,13 +26,56 @@ export class RetryRequestError extends Error {
     }
 }
 /**
- * Errors of `SessionError` type will trigger a session rotation.
+ * Errors of `SessionError` type retire the session associated with the request and trigger a regular retry.
  *
- * This error doesn't respect the `maxRequestRetries` option and has a separate limit of `maxSessionRotations`.
+ * The retry counts towards the `maxRequestRetries` limit, just like any other error.
  */
-export class SessionError extends RetryRequestError {
+export class SessionError extends Error {
     constructor(message) {
-        super(`Detected a session error, rotating session... ${message ? `\n${message}` : ''}`);
+        super(`Detected a session error, retiring session... ${message ? `\n${message}` : ''}`);
     }
 }
+/**
+ * Thrown when a requested session is not found in the referenced SessionPool.
+ */
+export class MissingSessionError extends Error {
+    constructor(sessionId) {
+        super(`The current SessionPool instance couldn't find a valid session${sessionId ? ` for the following id: ${sessionId}.` : '.'}`);
+    }
+}
+export class ContextPipelineInterruptedError extends Error {
+    constructor(message) {
+        super(`Request handling was interrupted during context initialization ${message ? ` - ${message}` : ''}`);
+    }
+}
+export class ContextPipelineInitializationError extends Error {
+    constructor(error, options) {
+        super(undefined, { cause: error, ...options });
+    }
+}
+export class ContextPipelineCleanupError extends CriticalError {
+    constructor(error, options) {
+        super(undefined, { cause: error, ...options });
+    }
+}
+export class RequestHandlerError extends Error {
+    constructor(error, options) {
+        super(undefined, { cause: error, ...options });
+    }
+}
+/**
+ * Thrown when attempting to set a different service instance after one has already been retrieved.
+ */
+export class ServiceConflictError extends Error {
+    constructor(serviceName, newValue, existingValue) {
+        super(`Service ${serviceName} is already in use. ` +
+            `Existing value: ${inspectValue(existingValue)}, attempted new value: ${inspectValue(newValue)}.`);
+    }
+}
+/**
+ * Thrown by crawlers when `skipNavigation` is used on a request.
+ * Subclasses can catch this error to skip their own navigation-dependent logic.
+ */
+export class NavigationSkippedError extends NonRetryableError {
+}
 //# sourceMappingURL=errors.js.map

package/errors.js.map

@@ -1 +1 @@
-{"version":3,"file":"errors.js","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,OAAO,iBAAkB,SAAQ,KAAK;CAAG;AAE/C;;;GAGG;AACH,MAAM,OAAO,aAAc,SAAQ,iBAAiB;CAAG;AAEvD;;GAEG;AACH,MAAM,OAAO,iBAAkB,SAAQ,aAAa;CAAG;AAEvD;;;;GAIG;AACH,MAAM,OAAO,iBAAkB,SAAQ,KAAK;IACxC,YAAY,OAAgB;QACxB,KAAK,CAAC,OAAO,IAAI,gDAAgD,CAAC,CAAC;IACvE,CAAC;CACJ;AAED;;;;GAIG;AACH,MAAM,OAAO,YAAa,SAAQ,iBAAiB;IAC/C,YAAY,OAAgB;QACxB,KAAK,CAAC,iDAAiD,OAAO,CAAC,CAAC,CAAC,KAAK,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC5F,CAAC;CACJ"}
+{"version":3,"file":"errors.js","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE9C;;GAEG;AACH,MAAM,OAAO,iBAAkB,SAAQ,KAAK;CAAG;AAE/C;;;GAGG;AACH,MAAM,OAAO,aAAc,SAAQ,iBAAiB;CAAG;AAEvD;;GAEG;AACH,MAAM,OAAO,iBAAkB,SAAQ,aAAa;CAAG;AAEvD;;;;GAIG;AACH,MAAM,OAAO,iBAAkB,SAAQ,KAAK;IACxC,YAAY,OAAgB;QACxB,KAAK,CAAC,OAAO,IAAI,gDAAgD,CAAC,CAAC;IACvE,CAAC;CACJ;AAED;;;;GAIG;AACH,MAAM,OAAO,YAAa,SAAQ,KAAK;IACnC,YAAY,OAAgB;QACxB,KAAK,CAAC,iDAAiD,OAAO,CAAC,CAAC,CAAC,KAAK,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC5F,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,OAAO,mBAAoB,SAAQ,KAAK;IAC1C,YAAY,SAAkB;QAC1B,KAAK,CACD,iEAAiE,SAAS,CAAC,CAAC,CAAC,0BAA0B,SAAS,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,CAC9H,CAAC;IACN,CAAC;CACJ;AAED,MAAM,OAAO,+BAAgC,SAAQ,KAAK;IACtD,YAAY,OAAgB;QACxB,KAAK,CAAC,kEAAkE,OAAO,CAAC,CAAC,CAAC,MAAM,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC9G,CAAC;CACJ;AAED,MAAM,OAAO,kCAAmC,SAAQ,KAAK;IACzD,YAAY,KAAc,EAAE,OAAsB;QAC9C,KAAK,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACnD,CAAC;CACJ;AAED,MAAM,OAAO,2BAA4B,SAAQ,aAAa;IAC1D,YAAY,KAAc,EAAE,OAAsB;QAC9C,KAAK,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACnD,CAAC;CACJ;AAED,MAAM,OAAO,mBAAoB,SAAQ,KAAK;IAC1C,YAAY,KAAc,EAAE,OAAsB;QAC9C,KAAK,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACnD,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IAC3C,YAAY,WAAmB,EAAE,QAAiB,EAAE,aAAsB;QACtE,KAAK,CACD,WAAW,WAAW,sBAAsB;YACxC,mBAAmB,YAAY,CAAC,aAAa,CAAC,0BAA0B,YAAY,CAAC,QAAQ,CAAC,GAAG,CACxG,CAAC;IACN,CAAC;CACJ;AAED;;;GAGG;AACH,MAAM,OAAO,sBAAuB,SAAQ,iBAAiB;CAAG"}

package/events/event_manager.d.ts

@@ -1,28 +1,55 @@
 import { AsyncEventEmitter } from '@vladfrangu/async_event_emitter';
 import type { BetterIntervalID } from '@apify/utilities';
-import { Configuration } from '../configuration.js';
+export interface EventManagerOptions {
+    /** Interval between emitted `persistState` events in milliseconds. */
+    persistStateIntervalMillis: number;
+}
 export declare const enum EventType {
     PERSIST_STATE = "persistState",
     SYSTEM_INFO = "systemInfo",
     MIGRATING = "migrating",
     ABORTING = "aborting",
-    EXIT = "exit"
+    EXIT = "exit",
+    STATUS_MESSAGE = "statusMessage"
+}
+export type EventTypeName = EventType | 'systemInfo' | 'persistState' | 'migrating' | 'aborting' | 'exit' | 'statusMessage';
+/**
+ * Payload emitted with the {@link EventType.STATUS_MESSAGE|`statusMessage`} event.
+ *
+ * The crawler broadcasts these whenever it wants to report its progress (e.g. periodically, or on
+ * start/finish). Consumers such as the Apify SDK can listen for the event and propagate the message
+ * to the platform. This keeps the crawler decoupled from any specific status-reporting backend.
+ */
+export interface EventStatusMessageData {
+    /**
+     * Identifies the crawler that emitted the message.
+     *
+     * Either the user-provided `id` from the crawler options, or a randomly generated one.
+     * Since a single event manager may be shared by multiple crawlers, consumers can use this
+     * to attribute the message to a specific crawler instance.
+     */
+    crawlerId: string;
+    /** The human-readable status message. */
+    message: string;
+    /** Whether this is the final status message of the run. */
+    isStatusMessageTerminal?: boolean;
+    /** The log level the message was logged with. */
+    level?: 'DEBUG' | 'INFO' | 'WARNING' | 'ERROR';
 }
-export type EventTypeName = EventType | 'systemInfo' | 'persistState' | 'migrating' | 'aborting' | 'exit';
 interface Intervals {
     persistState?: BetterIntervalID;
     systemInfo?: BetterIntervalID;
 }
 export declare abstract class EventManager {
-    readonly config: Configuration;
     protected events: AsyncEventEmitter<{}>;
     protected initialized: boolean;
     protected intervals: Intervals;
 // @ts-ignore optional peer dependency or compatibility with es2022
-    protected log: import("@apify/log").Log;
-    constructor(config?: Configuration);
+    protected log: import("@crawlee/types").CrawleeLogger;
+    private persistStateIntervalMillis;
+    constructor(options: EventManagerOptions);
     /**
-     * Initializes the event manager by creating the `persistState` event interval.
+     * Initializes the event manager by starting the `persistState` event interval.
      * This is automatically called at the beginning of `crawler.run()`.
      */
     init(): Promise<void>;

package/events/event_manager.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"event_manager.d.ts","sourceRoot":"","sources":["../../src/events/event_manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AAGpE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAGzD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,0BAAkB,SAAS;IACvB,aAAa,iBAAiB;IAC9B,WAAW,eAAe;IAC1B,SAAS,cAAc;IACvB,QAAQ,aAAa;IACrB,IAAI,SAAS;CAChB;AAED,MAAM,MAAM,aAAa,GAAG,SAAS,GAAG,YAAY,GAAG,cAAc,GAAG,WAAW,GAAG,UAAU,GAAG,MAAM,CAAC;AAE1G,UAAU,SAAS;IACf,YAAY,CAAC,EAAE,gBAAgB,CAAC;IAChC,UAAU,CAAC,EAAE,gBAAgB,CAAC;CACjC;AAED,8BAAsB,YAAY;IAMlB,QAAQ,CAAC,MAAM;IAL3B,SAAS,CAAC,MAAM,wBAA2B;IAC3C,SAAS,CAAC,WAAW,UAAS;IAC9B,SAAS,CAAC,SAAS,EAAE,SAAS,CAAM;IACpC,SAAS,CAAC,GAAG,2BAAmC;gBAE3B,MAAM,gBAAkC;IAI7D;;;OAGG;IACG,IAAI;IAaV;;;OAGG;IACG,KAAK;IAeX,EAAE,CAAC,KAAK,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,GAAG,IAAI;IAIjE,GAAG,CAAC,KAAK,EAAE,aAAa,EAAE,QAAQ,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,GAAG,IAAI;IAQnE,IAAI,CAAC,KAAK,EAAE,aAAa,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;IAIpD,aAAa,IAAI,OAAO;IAIxB;;OAEG;IACH,aAAa,CAAC,KAAK,EAAE,aAAa,GAAG,MAAM;IAI3C;;OAEG;IACH,SAAS,CAAC,KAAK,EAAE,aAAa,GAAG,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC,EAAE;IAI3D;;OAEG;IACG,6BAA6B;CAGtC"}
+{"version":3,"file":"event_manager.d.ts","sourceRoot":"","sources":["../../src/events/event_manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AAEpE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAKzD,MAAM,WAAW,mBAAmB;IAChC,sEAAsE;IACtE,0BAA0B,EAAE,MAAM,CAAC;CACtC;AAED,0BAAkB,SAAS;IACvB,aAAa,iBAAiB;IAC9B,WAAW,eAAe;IAC1B,SAAS,cAAc;IACvB,QAAQ,aAAa;IACrB,IAAI,SAAS;IACb,cAAc,kBAAkB;CACnC;AAED,MAAM,MAAM,aAAa,GACnB,SAAS,GACT,YAAY,GACZ,cAAc,GACd,WAAW,GACX,UAAU,GACV,MAAM,GACN,eAAe,CAAC;AAEtB;;;;;;GAMG;AACH,MAAM,WAAW,sBAAsB;IACnC;;;;;;OAMG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB,yCAAyC;IACzC,OAAO,EAAE,MAAM,CAAC;IAChB,2DAA2D;IAC3D,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAClC,iDAAiD;IACjD,KAAK,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC;CAClD;AAED,UAAU,SAAS;IACf,YAAY,CAAC,EAAE,gBAAgB,CAAC;IAChC,UAAU,CAAC,EAAE,gBAAgB,CAAC;CACjC;AAED,8BAAsB,YAAY;IAC9B,SAAS,CAAC,MAAM,wBAA2B;IAC3C,SAAS,CAAC,WAAW,UAAS;IAC9B,SAAS,CAAC,SAAS,EAAE,SAAS,CAAM;IACpC,SAAS,CAAC,GAAG,yCAA0D;IACvE,OAAO,CAAC,0BAA0B,CAAS;gBAE/B,OAAO,EAAE,mBAAmB;IAKxC;;;OAGG;IACG,IAAI;IAaV;;;OAGG;IACG,KAAK;IAeX,EAAE,CAAC,KAAK,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,GAAG,IAAI;IAIjE,GAAG,CAAC,KAAK,EAAE,aAAa,EAAE,QAAQ,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,GAAG,IAAI;IAQnE,IAAI,CAAC,KAAK,EAAE,aAAa,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;IAIpD,aAAa,IAAI,OAAO;IAIxB;;OAEG;IACH,aAAa,CAAC,KAAK,EAAE,aAAa,GAAG,MAAM;IAI3C;;OAEG;IACH,SAAS,CAAC,KAAK,EAAE,aAAa,GAAG,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC,EAAE;IAI3D;;OAEG;IACG,6BAA6B;CAGtC"}