@crawlee/core 4.0.0-beta.8 → 4.0.0-beta.80

package/crawlers/statistics.js

@@ -1,6 +1,5 @@
 import ow from 'ow';
-import { Configuration } from '../configuration.js';
-import { log as defaultLog } from '../log.js';
+import { serviceLocator } from '../service_locator.js';
 import { KeyValueStore } from '../storages/key_value_store.js';
 import { ErrorTracker } from './error_tracker.js';
 /**
@@ -30,7 +29,7 @@ const errorTrackerConfig = {
  * statistics for requests.
  *
  * All statistic information is saved on key value store
- * under the key `SDK_CRAWLER_STATISTICS_*`, persists between
+ * under the key `CRAWLEE_CRAWLER_STATISTICS_*`, persists between
  * migrations and abort/resurrect
  *
  * @category Crawlers
@@ -48,7 +47,7 @@ export class Statistics {
     /**
      * Statistic instance id.
      */
-    id = Statistics.id++; // assign an id while incrementing so it can be saved/restored from KV
+    id;
     /**
      * Current statistic state used for doing calculations on {@link Statistics.calculate} calls
      */
@@ -57,12 +56,8 @@ export class Statistics {
      * Contains the current retries histogram. Index 0 means 0 retries, index 2, 2 retries, and so on
      */
     requestRetryHistogram = [];
-    /**
-     * Contains the associated Configuration instance
-     */
-    config;
     keyValueStore = undefined;
-    persistStateKey = `SDK_CRAWLER_STATISTICS_${this.id}`;
+    persistStateKey;
     logIntervalMillis;
     logMessage;
     listener;
@@ -70,8 +65,14 @@ export class Statistics {
     log;
     instanceStart;
     logInterval;
-    events;
+    _events;
     persistenceOptions;
+    get events() {
+        if (!this._events) {
+            this._events = serviceLocator.getEventManager();
+        }
+        return this._events;
+    }
     /**
      * @internal
      */
@@ -81,22 +82,22 @@ export class Statistics {
             logMessage: ow.optional.string,
             log: ow.optional.object,
             keyValueStore: ow.optional.object,
-            config: ow.optional.object,
             persistenceOptions: ow.optional.object,
             saveErrorSnapshots: ow.optional.boolean,
+            id: ow.optional.any(ow.number, ow.string),
         }));
-        const { logIntervalSecs = 60, logMessage = 'Statistics', keyValueStore, config = Configuration.getGlobalConfig(), persistenceOptions = {
+        const { logIntervalSecs = 60, logMessage = 'Statistics', keyValueStore, persistenceOptions = {
             enable: true,
-        }, saveErrorSnapshots = false, } = options;
-        this.log = (options.log ?? defaultLog).child({ prefix: 'Statistics' });
+        }, saveErrorSnapshots = false, id, } = options;
+        this.id = id ?? String(Statistics.id++);
+        this.persistStateKey = `CRAWLEE_CRAWLER_STATISTICS_${this.id}`;
+        this.log = (options.log ?? serviceLocator.getLogger()).child({ prefix: 'Statistics' });
         this.errorTracker = new ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });
         this.errorTrackerRetry = new ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });
         this.logIntervalMillis = logIntervalSecs * 1000;
         this.logMessage = logMessage;
         this.keyValueStore = keyValueStore;
         this.listener = this.persistState.bind(this);
-        this.events = config.getEventManager();
-        this.config = config;
         this.persistenceOptions = persistenceOptions;
         // initialize by "resetting"
         this.reset();
@@ -194,6 +195,14 @@ export class Statistics {
         this._saveRetryCountForJob(retryCount);
         this.requestsInProgress.delete(id);
     }
+    /**
+     * Discards a started job without affecting the finished/failed counters, e.g. when a request
+     * turns out to be skipped (robots.txt, enqueue strategy) after `startJob` was already called for it.
+     * @ignore
+     */
+    discardJob(id) {
+        this.requestsInProgress.delete(id);
+    }
     /**
      * Calculate the current statistics
      */
@@ -216,7 +225,7 @@ export class Statistics {
      * displaying the current state in predefined intervals
      */
     async startCapturing() {
-        this.keyValueStore ??= await KeyValueStore.open(null, { config: this.config });
+        this.keyValueStore ??= await KeyValueStore.open(null, { config: serviceLocator.getConfiguration() });
         if (this.state.crawlerStartedAt === null) {
             this.state.crawlerStartedAt = new Date();
         }
@@ -258,14 +267,8 @@ export class Statistics {
             return;
         }
         this.log.debug('Persisting state', { persistStateKey: this.persistStateKey });
-        // use half the interval of `persistState` to avoid race conditions
-        const persistStateIntervalMillis = this.config.get('persistStateIntervalMillis');
-        const timeoutSecs = persistStateIntervalMillis / 2_000;
         await this.keyValueStore
-            .setValue(this.persistStateKey, this.toJSON(), {
-            timeoutSecs,
-            doNotRetryTimeouts: true,
-        })
+            .setValue(this.persistStateKey, this.toJSON())
             .catch((error) => this.log.warning(`Failed to persist the statistics to ${this.persistStateKey}`, { error }));
     }
     /**
@@ -308,7 +311,9 @@ export class Statistics {
     }
     _teardown() {
         // this can be called before a call to startCapturing happens (or in a 'finally' block)
-        this.events.off("persistState" /* EventType.PERSIST_STATE */, this.listener);
+        // Only unsubscribe if event manager was already resolved — avoid eagerly resolving it
+        // (e.g. during the constructor's reset() call, which would capture the wrong context)
+        this._events?.off("persistState" /* EventType.PERSIST_STATE */, this.listener);
         if (this.logInterval) {
             clearInterval(this.logInterval);
             this.logInterval = null;
@@ -343,4 +348,3 @@ export class Statistics {
         return result;
     }
 }
-//# sourceMappingURL=statistics.js.map

package/enqueue_links/enqueue_links.d.ts

@@ -1,25 +1,30 @@
-import type { Awaitable, BatchAddRequestsResult, Dictionary } from '@crawlee/types';
+import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
 import { type RobotsTxtFile } from '@crawlee/utils';
 import type { SetRequired } from 'type-fest';
-import type { RequestProvider, RequestQueueOperationOptions } from '../storages/request_provider.js';
-import type { GlobInput, PseudoUrlInput, RegExpInput, RequestTransform } from './shared.js';
-export type SkippedRequestCallback = (args: {
-    url: string;
-    reason: 'robotsTxt';
-}) => Awaitable<void>;
+import { Request } from '../request.js';
+import type { IRequestManager } from '../storages/request_manager.js';
+import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, RequestQueueOperationOptions } from '../storages/request_queue.js';
+import type { GlobInput, PseudoUrlInput, RegExpInput, RequestTransform, SkippedRequestCallback } from './shared.js';
 export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
     /** Limit the amount of actually enqueued URLs to this number. Useful for testing across the entire crawling scope. */
     limit?: number;
     /** An array of URLs to enqueue. */
     urls?: readonly string[];
-    /** A request queue to which the URLs will be enqueued. */
-    requestQueue?: RequestProvider;
+    /** A request manager to which the URLs will be enqueued. */
+    requestManager?: IRequestManager;
     /** A CSS selector matching links to be enqueued. */
     selector?: string;
     /** Sets {@link Request.userData} for newly enqueued requests. */
     userData?: Dictionary;
-    /** Sets {@link Request.label} for newly enqueued requests. */
+    /**
+     * Sets {@link Request.label} for newly enqueued requests.
+     *
+     * This option has the lowest priority and can be overwritten by request options
+     * specified in `globs`, `regexps`, or `pseudoUrls` objects, as well as by `transformRequestFunction`.
+     */
     label?: string;
+    /** Sets {@link Request.sessionId} for newly enqueued requests. */
+    sessionId?: string;
     /**
      * If set to `true`, tells the crawler to skip navigation and process the request directly.
      * @default false
@@ -49,7 +54,6 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
      * containing patterns matching URLs that will **never** be enqueued.
      *
      * The plain objects must include either the `glob` property or the `regexp` property.
-     * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
      *
      * Glob matching is always case-insensitive.
      * If you need case-sensitive matching, provide a regexp.
@@ -86,12 +90,12 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
      */
     pseudoUrls?: readonly PseudoUrlInput[];
     /**
-     * Just before a new {@link Request} is constructed and enqueued to the {@link RequestQueue}, this function can be used
-     * to remove it or modify its contents such as `userData`, `payload` or, most importantly `uniqueKey`. This is useful
+     * After request options are filtered by patterns, this function can be used
+     * to remove them or modify their contents such as `userData`, `payload` or, most importantly `uniqueKey`. This is useful
      * when you need to enqueue multiple `Requests` to the queue that share the same URL, but differ in methods or payloads,
      * or to dynamically update or create `userData`.
      *
-     * For example: by adding `keepUrlFragment: true` to the `request` object, URL fragments will not be removed
+     * For example: by adding `keepUrlFragment: true` to the request options, URL fragments will not be removed
      * when `uniqueKey` is computed.
      *
      * **Example:**
@@ -105,9 +109,13 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
      * }
      * ```
      *
-     * Note that `transformRequestFunction` has a priority over request options
-     * specified in `globs`, `regexps`, or `pseudoUrls` objects,
-     * and thus some options could be over-written by `transformRequestFunction`.
+     * Note that `transformRequestFunction` has the highest priority and can overwrite request options
+     * specified in `globs`, `regexps`, or `pseudoUrls` objects, as well as the global `label` option.
+     *
+     * The function receives a {@link RequestOptions} object and can return either:
+     * - The modified {@link RequestOptions} object
+     * - `'unchanged'` to keep the original options as-is
+     * - A falsy value or `'skip'` to exclude the request from the queue
      */
     transformRequestFunction?: RequestTransform;
     /**
@@ -138,10 +146,21 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
      * RobotsTxtFile instance for the current request that triggered the `enqueueLinks`.
      * If provided, disallowed URLs will be ignored.
      */
-    robotsTxtFile?: RobotsTxtFile;
+    robotsTxtFile?: Pick<RobotsTxtFile, 'isAllowed'>;
+    /**
+     * Mirrors {@link BasicCrawlerOptions.respectRobotsTxtFile}: pass `false` to disable filtering or
+     * `{ userAgent }` to evaluate rules for a specific user-agent. Defaults to `*` when
+     * {@link EnqueueLinksOptions.robotsTxtFile|`robotsTxtFile`} is provided.
+     */
+    respectRobotsTxtFile?: boolean | {
+        userAgent?: string;
+    };
     /**
      * When a request is skipped for some reason, you can use this callback to act on it.
-     * This is currently fired only for requests skipped based on robots.txt file.
+     * This is currently fired for requests skipped
+     * 1. based on robots.txt file,
+     * 2. because they don't match enqueueLinks filters,
+     * 3. or because the maxRequestsPerCrawl limit has been reached
      */
     onSkippedRequest?: SkippedRequestCallback;
 }
@@ -208,7 +227,7 @@ export declare enum EnqueueStrategy {
  * ```javascript
  * await enqueueLinks({
  *   urls: aListOfFoundUrls,
- *   requestQueue,
+ *   requestManager,
  *   selector: 'a.product-detail',
  *   globs: [
  *       'https://www.example.com/handbags/*',
@@ -220,7 +239,11 @@ export declare enum EnqueueStrategy {
  * @param options All `enqueueLinks()` parameters are passed via an options object.
  * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
  */
-export declare function enqueueLinks(options: SetRequired<EnqueueLinksOptions, 'requestQueue' | 'urls'>): Promise<BatchAddRequestsResult>;
+export declare function enqueueLinks(options: SetRequired<Omit<EnqueueLinksOptions, 'requestManager'>, 'urls'> & {
+    requestManager: {
+        addRequestsBatched: (requests: Request<Dictionary>[], options: AddRequestsBatchedOptions) => Promise<AddRequestsBatchedResult>;
+    };
+}): Promise<BatchAddRequestsResult>;
 /**
  * @internal
  * This method helps resolve the baseUrl that will be used for filtering in {@link enqueueLinks}.
@@ -239,4 +262,3 @@ export interface ResolveBaseUrl {
     originalRequestUrl: string;
     finalRequestUrl?: string;
 }
-//# sourceMappingURL=enqueue_links.d.ts.map

package/enqueue_links/enqueue_links.js

@@ -1,7 +1,8 @@
 import ow from 'ow';
 import { getDomain } from 'tldts';
-import log from '@apify/log';
-import { constructGlobObjectsFromGlobs, constructRegExpObjectsFromPseudoUrls, constructRegExpObjectsFromRegExps, createRequestOptions, createRequests, filterRequestsByPatterns, } from './shared.js';
+import { Request } from '../request.js';
+import { serviceLocator } from '../service_locator.js';
+import { applyRequestTransform, constructGlobObjectsFromGlobs, constructRegExpObjectsFromPseudoUrls, constructRegExpObjectsFromRegExps, createRequestOptions, filterRequestOptionsByPatterns, } from './shared.js';
 /**
  * The different enqueueing strategies available.
  *
@@ -66,7 +67,7 @@ export var EnqueueStrategy;
  * ```javascript
  * await enqueueLinks({
  *   urls: aListOfFoundUrls,
- *   requestQueue,
+ *   requestManager,
  *   selector: 'a.product-detail',
  *   globs: [
  *       'https://www.example.com/handbags/*',
@@ -87,11 +88,13 @@ export async function enqueueLinks(options) {
     }
     ow(options, ow.object.exactShape({
         urls: ow.array.ofType(ow.string),
-        requestQueue: ow.object.hasKeys('fetchNextRequest', 'addRequest'),
+        requestManager: ow.object.hasKeys('addRequestsBatched'),
         robotsTxtFile: ow.optional.object.hasKeys('isAllowed'),
+        respectRobotsTxtFile: ow.optional.any(ow.boolean, ow.object.exactShape({ userAgent: ow.optional.string })),
         onSkippedRequest: ow.optional.function,
         forefront: ow.optional.boolean,
         skipNavigation: ow.optional.boolean,
+        sessionId: ow.optional.string,
         limit: ow.optional.number,
         selector: ow.optional.string,
         baseUrl: ow.optional.string,
@@ -105,7 +108,9 @@ export async function enqueueLinks(options) {
         strategy: ow.optional.string.oneOf(Object.values(EnqueueStrategy)),
         waitForAllRequestsToBeAdded: ow.optional.boolean,
     }));
-    const { requestQueue, limit, urls, pseudoUrls, exclude, globs, regexps, transformRequestFunction, forefront, waitForAllRequestsToBeAdded, robotsTxtFile, onSkippedRequest, } = options;
+    const { requestManager, limit, urls, 
+    // oxlint-disable-next-line typescript/no-deprecated -- still accepted for backwards compat
+    pseudoUrls, exclude, globs, regexps, transformRequestFunction, forefront, waitForAllRequestsToBeAdded, robotsTxtFile, onSkippedRequest, } = options;
     const urlExcludePatternObjects = [];
     const urlPatternObjects = [];
     if (exclude?.length) {
@@ -119,7 +124,7 @@ export async function enqueueLinks(options) {
         }
     }
     if (pseudoUrls?.length) {
-        log.deprecated('`pseudoUrls` option is deprecated, use `globs` or `regexps` instead');
+        serviceLocator.getLogger().deprecated('`pseudoUrls` option is deprecated, use `globs` or `regexps` instead');
         urlPatternObjects.push(...constructRegExpObjectsFromPseudoUrls(pseudoUrls));
     }
     if (globs?.length) {
@@ -167,44 +172,61 @@ export async function enqueueLinks(options) {
                 break;
         }
     }
+    async function reportSkippedRequests(skippedRequests, reason) {
+        if (onSkippedRequest && skippedRequests.length > 0) {
+            await Promise.all(skippedRequests.map((request) => {
+                return onSkippedRequest({
+                    url: request.url,
+                    reason: request.skippedReason ?? reason,
+                });
+            }));
+        }
+    }
     let requestOptions = createRequestOptions(urls, options);
-    if (robotsTxtFile) {
+    if (robotsTxtFile && options.respectRobotsTxtFile !== false) {
+        const robotsUserAgent = typeof options.respectRobotsTxtFile === 'object' ? (options.respectRobotsTxtFile.userAgent ?? '*') : '*';
         const skippedRequests = [];
         requestOptions = requestOptions.filter((request) => {
-            if (robotsTxtFile.isAllowed(request.url)) {
+            if (robotsTxtFile.isAllowed(request.url, robotsUserAgent)) {
                 return true;
             }
             skippedRequests.push(request);
             return false;
         });
-        if (onSkippedRequest && skippedRequests.length > 0) {
-            await Promise.all(skippedRequests.map((request) => {
-                return onSkippedRequest({ url: request.url, reason: 'robotsTxt' });
-            }));
-        }
+        await reportSkippedRequests(skippedRequests, 'robotsTxt');
     }
-    if (transformRequestFunction) {
-        requestOptions = requestOptions
-            .map((request) => transformRequestFunction(request))
-            .filter((r) => !!r);
-    }
-    function createFilteredRequests() {
-        // No user provided patterns means we can skip an extra filtering step
+    async function createFilteredRequests() {
+        const skippedRequests = [];
+        // Step 1: Filter request options by exclude patterns, user patterns (globs/regexps), and strategy patterns.
+        // Pattern-level options (label, userData, method, etc.) are merged during this step.
+        let filteredOptions;
         if (urlPatternObjects.length === 0) {
-            return createRequests(requestOptions, enqueueStrategyPatterns, urlExcludePatternObjects, options.strategy);
+            filteredOptions = filterRequestOptionsByPatterns(requestOptions, enqueueStrategyPatterns.length > 0 ? enqueueStrategyPatterns : undefined, urlExcludePatternObjects, options.strategy, (url) => skippedRequests.push(url));
         }
-        // Generate requests based on the user patterns first
-        const generatedRequestsFromUserFilters = createRequests(requestOptions, urlPatternObjects, urlExcludePatternObjects, options.strategy);
-        // ...then filter them by the enqueue links strategy (making this an AND check)
-        return filterRequestsByPatterns(generatedRequestsFromUserFilters, enqueueStrategyPatterns);
+        else {
+            // Filter by user patterns first (with exclude)
+            const afterUserPatterns = filterRequestOptionsByPatterns(requestOptions, urlPatternObjects, urlExcludePatternObjects, options.strategy, (url) => skippedRequests.push(url));
+            // ...then filter by the enqueue links strategy (making this an AND check)
+            filteredOptions = filterRequestOptionsByPatterns(afterUserPatterns, enqueueStrategyPatterns.length > 0 ? enqueueStrategyPatterns : undefined, [], options.strategy, (url) => skippedRequests.push(url));
+        }
+        await reportSkippedRequests(skippedRequests.map((url) => ({ url })), 'filters');
+        // Step 2: Apply transformRequestFunction on request options - it has the highest priority
+        if (transformRequestFunction) {
+            const skippedByTransform = [];
+            filteredOptions = applyRequestTransform(filteredOptions, transformRequestFunction, (r) => skippedByTransform.push(r));
+            await reportSkippedRequests(skippedByTransform, 'transform');
+        }
+        // Step 3: Create Request instances from the final request options
+        return filteredOptions.map((opts) => new Request(opts));
     }
-    let requests = createFilteredRequests();
-    if (limit)
-        requests = requests.slice(0, limit);
-    const { addedRequests } = await requestQueue.addRequestsBatched(requests, {
+    const { addedRequests, requestsOverLimit } = await requestManager.addRequestsBatched(await createFilteredRequests(), {
         forefront,
         waitForAllRequestsToBeAdded,
+        maxNewRequests: limit,
     });
+    if (requestsOverLimit?.length !== undefined && requestsOverLimit.length > 0) {
+        await reportSkippedRequests(requestsOverLimit.map((r) => ({ url: typeof r === 'string' ? r : r.url })), 'enqueueLimit');
+    }
     return { processedRequests: addedRequests, unprocessedRequests: [] };
 }
 /**
@@ -247,4 +269,3 @@ export function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy, finalR
 function ignoreHttpSchema(pattern) {
     return pattern.replace(/^(https?):\/\//, 'http{s,}://');
 }
-//# sourceMappingURL=enqueue_links.js.map

package/enqueue_links/index.d.ts

@@ -1,3 +1,2 @@
 export * from './enqueue_links.js';
 export * from './shared.js';
-//# sourceMappingURL=index.d.ts.map

package/enqueue_links/index.js

@@ -1,3 +1,2 @@
 export * from './enqueue_links.js';
 export * from './shared.js';
-//# sourceMappingURL=index.js.map

package/enqueue_links/shared.d.ts

@@ -1,5 +1,5 @@
+import type { Awaitable } from '@crawlee/types';
 import type { RequestOptions } from '../request.js';
-import { Request } from '../request.js';
 import type { EnqueueLinksOptions } from './enqueue_links.js';
 export { tryAbsoluteURL } from '@crawlee/utils';
 export type UrlPatternObject = {
@@ -18,6 +18,11 @@ export type RegExpObject = {
     regexp: RegExp;
 } & Pick<RequestOptions, 'method' | 'payload' | 'label' | 'userData' | 'headers'>;
 export type RegExpInput = RegExp | RegExpObject;
+export type SkippedRequestReason = 'robotsTxt' | 'limit' | 'enqueueLimit' | 'filters' | 'transform' | 'redirect' | 'depth';
+export type SkippedRequestCallback = (args: {
+    url: string;
+    reason: SkippedRequestReason;
+}) => Awaitable<void>;
 /**
  * @ignore
  */
@@ -45,23 +50,34 @@ export declare function validateGlobPattern(glob: string): string;
  */
 export declare function constructRegExpObjectsFromRegExps(regexps: readonly RegExpInput[]): RegExpObject[];
 /**
+ * Filters request options by URL patterns and merges pattern-level options (label, userData, method, payload, headers)
+ * from the first matching pattern into each RequestOptions entry.
+ *
+ * When `includePatterns` is empty/undefined, all options pass through (only exclude filtering applies).
  * @ignore
  */
-export declare function createRequests(requestOptions: (string | RequestOptions)[], urlPatternObjects?: UrlPatternObject[], excludePatternObjects?: UrlPatternObject[], strategy?: EnqueueLinksOptions['strategy']): Request[];
-export declare function filterRequestsByPatterns(requests: Request[], patterns?: UrlPatternObject[]): Request[];
+export declare function filterRequestOptionsByPatterns(requestOptions: RequestOptions[], includePatterns: UrlPatternObject[] | undefined, excludePatterns?: UrlPatternObject[], strategy?: EnqueueLinksOptions['strategy'], onSkippedUrl?: (url: string) => void): RequestOptions[];
 /**
  * @ignore
  */
-export declare function createRequestOptions(sources: readonly (string | Record<string, unknown>)[], options?: Pick<EnqueueLinksOptions, 'label' | 'userData' | 'baseUrl' | 'skipNavigation' | 'strategy'>): RequestOptions[];
+export declare function createRequestOptions(sources: readonly (string | Record<string, unknown>)[], options?: Pick<EnqueueLinksOptions, 'label' | 'userData' | 'baseUrl' | 'skipNavigation' | 'sessionId' | 'strategy'>): RequestOptions[];
 /**
- * Takes an Apify {@link RequestOptions} object and changes its attributes in a desired way. This user-function is used
- * {@link enqueueLinks} to modify requests before enqueuing them.
+ * Takes a {@link RequestOptions} object and changes its attributes in a desired way. This user-function is used
+ * by {@link enqueueLinks} to modify request options before they are converted to {@link Request} instances.
  */
 export interface RequestTransform {
     /**
      * @param original Request options to be modified.
-     * @returns The modified request options to enqueue.
+     * @returns The modified request options to enqueue, `'unchanged'` to keep the original options as-is,
+     *   or a falsy value / `'skip'` to exclude the request from the queue.
      */
-    (original: RequestOptions): RequestOptions | false | undefined | null;
+    (original: RequestOptions): RequestOptions | false | undefined | null | 'skip' | 'unchanged';
 }
-//# sourceMappingURL=shared.d.ts.map
+/**
+ * Applies a {@link RequestTransform} function to a list of request options.
+ * Options for which the transform returns a falsy value are removed from the list.
+ * @param onSkipped Called with the original request options when the transform returns a falsy value (i.e. the request is skipped).
+ * @ignore
+ * @internal
+ */
+export declare function applyRequestTransform(requestOptions: RequestOptions[], transformFn: RequestTransform, onSkipped?: (requestOptions: RequestOptions) => void): RequestOptions[];

package/enqueue_links/shared.js

@@ -1,7 +1,6 @@
 import { URL } from 'node:url';
-import { minimatch } from 'minimatch';
+import { Minimatch } from 'minimatch';
 import { purlToRegExp } from '@apify/pseudo_url';
-import { Request } from '../request.js';
 export { tryAbsoluteURL } from '@crawlee/utils';
 const MAX_ENQUEUE_LINKS_CACHE_SIZE = 1000;
 /**
@@ -110,51 +109,37 @@ export function constructRegExpObjectsFromRegExps(regexps) {
     });
 }
 /**
+ * Filters request options by URL patterns and merges pattern-level options (label, userData, method, payload, headers)
+ * from the first matching pattern into each RequestOptions entry.
+ *
+ * When `includePatterns` is empty/undefined, all options pass through (only exclude filtering applies).
  * @ignore
  */
-export function createRequests(requestOptions, urlPatternObjects, excludePatternObjects = [], strategy) {
+export function filterRequestOptionsByPatterns(requestOptions, includePatterns, excludePatterns = [], strategy, onSkippedUrl) {
+    const excludeMatchers = excludePatterns.map(createPatternObjectMatcher);
+    const includeMatchers = includePatterns?.length ? includePatterns.map(createPatternObjectMatcher) : undefined;
     return requestOptions
-        .map((opts) => ({ url: typeof opts === 'string' ? opts : opts.url, opts }))
         .filter(({ url }) => {
-        return !excludePatternObjects.some((excludePatternObject) => {
-            const { regexp, glob } = excludePatternObject;
-            return (regexp && url.match(regexp)) || (glob && minimatch(url, glob, { nocase: true }));
-        });
+        const matchesExclude = excludeMatchers.some(({ match }) => match(url));
+        if (matchesExclude) {
+            onSkippedUrl?.(url);
+        }
+        return !matchesExclude;
     })
-        .map(({ url, opts }) => {
-        if (!urlPatternObjects || !urlPatternObjects.length) {
-            return new Request(typeof opts === 'string' ? { url: opts, enqueueStrategy: strategy } : { ...opts });
-        }
-        for (const urlPatternObject of urlPatternObjects) {
-            const { regexp, glob, ...requestRegExpOptions } = urlPatternObject;
-            if ((regexp && url.match(regexp)) || (glob && minimatch(url, glob, { nocase: true }))) {
-                const request = typeof opts === 'string'
-                    ? { url: opts, ...requestRegExpOptions, enqueueStrategy: strategy }
-                    : { ...opts, ...requestRegExpOptions, enqueueStrategy: strategy };
-                return new Request(request);
+        .map((opts) => {
+        if (!includeMatchers) {
+            return { ...opts, enqueueStrategy: strategy };
+        }
+        for (const { match, glob, regexp, ...patternOptions } of includeMatchers) {
+            if (match(opts.url)) {
+                return { ...opts, ...patternOptions, enqueueStrategy: strategy };
             }
         }
         // didn't match any positive pattern
+        onSkippedUrl?.(opts.url);
         return null;
     })
-        .filter((request) => request);
-}
-export function filterRequestsByPatterns(requests, patterns) {
-    if (!patterns?.length) {
-        return requests;
-    }
-    const filtered = [];
-    for (const request of requests) {
-        for (const urlPatternObject of patterns) {
-            const { regexp, glob } = urlPatternObject;
-            if ((regexp && request.url.match(regexp)) || (glob && minimatch(request.url, glob, { nocase: true }))) {
-                filtered.push(request);
-                // Break the pattern loop, as we already matched this request once
-                break;
-            }
-        }
-    }
-    return filtered;
+        .filter((opts) => opts !== null);
 }
 /**
  * @ignore
@@ -184,7 +169,53 @@ export function createRequestOptions(sources, options = {}) {
         if (options.skipNavigation) {
             requestOptions.skipNavigation = true;
         }
+        if (options.sessionId) {
+            requestOptions.sessionId = options.sessionId;
+        }
         return requestOptions;
     });
 }
-//# sourceMappingURL=shared.js.map
+/**
+ * @ignore
+ */
+function createPatternObjectMatcher(urlPatternObject) {
+    const { regexp, glob } = urlPatternObject;
+    let match;
+    if (regexp) {
+        match = (url) => regexp.test(url);
+    }
+    else if (glob) {
+        const m = new Minimatch(glob, { nocase: true });
+        match = (url) => m.match(url);
+    }
+    else {
+        match = () => false;
+    }
+    return { ...urlPatternObject, match };
+}
+/**
+ * Applies a {@link RequestTransform} function to a list of request options.
+ * Options for which the transform returns a falsy value are removed from the list.
+ * @param onSkipped Called with the original request options when the transform returns a falsy value (i.e. the request is skipped).
+ * @ignore
+ * @internal
+ */
+export function applyRequestTransform(requestOptions, transformFn, onSkipped) {
+    return requestOptions
+        .map((opts) => {
+        const transformed = transformFn(opts);
+        if (transformed === 'skip') {
+            onSkipped?.(opts);
+            return null;
+        }
+        if (transformed === 'unchanged') {
+            return opts;
+        }
+        if (!transformed) {
+            onSkipped?.(opts);
+            return null;
+        }
+        return transformed;
+    })
+        .filter((r) => r !== null);
+}