@crawlee/core 4.0.0-beta.15 → 4.0.0-beta.150

package/enqueue_links/enqueue_links.js

@@ -1,231 +1,6 @@
-import ow from 'ow';
+import { EnqueueStrategy } from '@crawlee/utils';
 import { getDomain } from 'tldts';
-import log from '@apify/log';
-import { constructGlobObjectsFromGlobs, constructRegExpObjectsFromPseudoUrls, constructRegExpObjectsFromRegExps, createRequestOptions, createRequests, filterRequestsByPatterns, } from './shared.js';
-/**
- * The different enqueueing strategies available.
- *
- * Depending on the strategy you select, we will only check certain parts of the URLs found. Here is a diagram of each URL part and their name:
- *
- * ```md
- * Protocol          Domain
- * ┌────┐          ┌─────────┐
- * https://example.crawlee.dev/...
- * │       └─────────────────┤
- * │             Hostname    │
- * │                         │
- * └─────────────────────────┘
- *          Origin
- *```
- *
- * - The `Protocol` is usually `http` or `https`
- * - The `Domain` represents the path without any possible subdomains to a website. For example, `crawlee.dev` is the domain of `https://example.crawlee.dev/`
- * - The `Hostname` is the full path to a website, including any subdomains. For example, `example.crawlee.dev` is the hostname of `https://example.crawlee.dev/`
- * - The `Origin` is the combination of the `Protocol` and `Hostname`. For example, `https://example.crawlee.dev` is the origin of `https://example.crawlee.dev/`
- */
-export var EnqueueStrategy;
-(function (EnqueueStrategy) {
-    /**
-     * Matches any URLs found
-     */
-    EnqueueStrategy["All"] = "all";
-    /**
-     * Matches any URLs that have the same hostname.
-     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
-     * `https://example.com/hello` will not be matched.
-     *
-     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
-     */
-    EnqueueStrategy["SameHostname"] = "same-hostname";
-    /**
-     * Matches any URLs that have the same domain as the base URL.
-     * For example, `https://wow.an.example.com` and `https://example.com` will both be matched for a base url of
-     * `https://example.com`.
-     *
-     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
-     */
-    EnqueueStrategy["SameDomain"] = "same-domain";
-    /**
-     * Matches any URLs that have the same hostname and protocol.
-     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
-     * `http://wow.example.com/hello` will not be matched.
-     *
-     * > This strategy will ensure the protocol of the base URL is the same as the protocol of the URL to be enqueued.
-     */
-    EnqueueStrategy["SameOrigin"] = "same-origin";
-})(EnqueueStrategy || (EnqueueStrategy = {}));
-/**
- * This function enqueues the urls provided to the {@link RequestQueue} provided. If you want to automatically find and enqueue links,
- * you should use the context-aware `enqueueLinks` function provided on the crawler contexts.
- *
- * Optionally, the function allows you to filter the target links' URLs using an array of globs or regular expressions
- * and override settings of the enqueued {@link Request} objects.
- *
- * **Example usage**
- *
- * ```javascript
- * await enqueueLinks({
- *   urls: aListOfFoundUrls,
- *   requestQueue,
- *   selector: 'a.product-detail',
- *   globs: [
- *       'https://www.example.com/handbags/*',
- *       'https://www.example.com/purses/*'
- *   ],
- * });
- * ```
- *
- * @param options All `enqueueLinks()` parameters are passed via an options object.
- * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
- */
-export async function enqueueLinks(options) {
-    if (!options || Object.keys(options).length === 0) {
-        throw new RangeError([
-            'enqueueLinks() was called without the required options. You can only do that when you use the `crawlingContext.enqueueLinks()` method in request handlers.',
-            'Check out our guide on how to use enqueueLinks() here: https://crawlee.dev/js/docs/examples/crawl-relative-links',
-        ].join('\n'));
-    }
-    ow(options, ow.object.exactShape({
-        urls: ow.array.ofType(ow.string),
-        requestQueue: ow.object.hasKeys('addRequestsBatched'),
-        robotsTxtFile: ow.optional.object.hasKeys('isAllowed'),
-        onSkippedRequest: ow.optional.function,
-        forefront: ow.optional.boolean,
-        skipNavigation: ow.optional.boolean,
-        limit: ow.optional.number,
-        selector: ow.optional.string,
-        baseUrl: ow.optional.string,
-        userData: ow.optional.object,
-        label: ow.optional.string,
-        pseudoUrls: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('purl'))),
-        globs: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('glob'))),
-        exclude: ow.optional.array.ofType(ow.any(ow.string, ow.regExp, ow.object.hasKeys('glob'), ow.object.hasKeys('regexp'))),
-        regexps: ow.optional.array.ofType(ow.any(ow.regExp, ow.object.hasKeys('regexp'))),
-        transformRequestFunction: ow.optional.function,
-        strategy: ow.optional.string.oneOf(Object.values(EnqueueStrategy)),
-        waitForAllRequestsToBeAdded: ow.optional.boolean,
-    }));
-    const { requestQueue, limit, urls, pseudoUrls, exclude, globs, regexps, transformRequestFunction, forefront, waitForAllRequestsToBeAdded, robotsTxtFile, onSkippedRequest, } = options;
-    const urlExcludePatternObjects = [];
-    const urlPatternObjects = [];
-    if (exclude?.length) {
-        for (const excl of exclude) {
-            if (typeof excl === 'string' || 'glob' in excl) {
-                urlExcludePatternObjects.push(...constructGlobObjectsFromGlobs([excl]));
-            }
-            else if (excl instanceof RegExp || 'regexp' in excl) {
-                urlExcludePatternObjects.push(...constructRegExpObjectsFromRegExps([excl]));
-            }
-        }
-    }
-    if (pseudoUrls?.length) {
-        log.deprecated('`pseudoUrls` option is deprecated, use `globs` or `regexps` instead');
-        urlPatternObjects.push(...constructRegExpObjectsFromPseudoUrls(pseudoUrls));
-    }
-    if (globs?.length) {
-        urlPatternObjects.push(...constructGlobObjectsFromGlobs(globs));
-    }
-    if (regexps?.length) {
-        urlPatternObjects.push(...constructRegExpObjectsFromRegExps(regexps));
-    }
-    if (!urlPatternObjects.length) {
-        options.strategy ??= EnqueueStrategy.SameHostname;
-    }
-    const enqueueStrategyPatterns = [];
-    if (options.baseUrl) {
-        const url = new URL(options.baseUrl);
-        switch (options.strategy) {
-            case EnqueueStrategy.SameHostname:
-                // We need to get the origin of the passed in domain in the event someone sets baseUrl
-                // to an url like https://example.com/deep/default/path and one of the found urls is an
-                // absolute relative path (/path/to/page)
-                enqueueStrategyPatterns.push({ glob: ignoreHttpSchema(`${url.origin}/**`) });
-                break;
-            case EnqueueStrategy.SameDomain: {
-                // Get the actual hostname from the base url
-                const baseUrlHostname = getDomain(url.hostname, { mixedInputs: false });
-                if (baseUrlHostname) {
-                    // We have a hostname, so we can use it to match all links on the page that point to it and any subdomains of it
-                    url.hostname = baseUrlHostname;
-                    enqueueStrategyPatterns.push({ glob: ignoreHttpSchema(`${url.origin.replace(baseUrlHostname, `*.${baseUrlHostname}`)}/**`) }, { glob: ignoreHttpSchema(`${url.origin}/**`) });
-                }
-                else {
-                    // We don't have a hostname (can happen for ips for instance), so reproduce the same behavior
-                    // as SameDomainAndSubdomain
-                    enqueueStrategyPatterns.push({ glob: ignoreHttpSchema(`${url.origin}/**`) });
-                }
-                break;
-            }
-            case EnqueueStrategy.SameOrigin: {
-                // The same behavior as SameHostname, but respecting the protocol of the URL
-                enqueueStrategyPatterns.push({ glob: `${url.origin}/**` });
-                break;
-            }
-            case EnqueueStrategy.All:
-            default:
-                enqueueStrategyPatterns.push({ glob: `http{s,}://**` });
-                break;
-        }
-    }
-    async function reportSkippedRequests(skippedRequests, reason) {
-        if (onSkippedRequest && skippedRequests.length > 0) {
-            await Promise.all(skippedRequests.map((request) => {
-                return onSkippedRequest({
-                    url: request.url,
-                    reason: request.skippedReason ?? reason,
-                });
-            }));
-        }
-    }
-    let requestOptions = createRequestOptions(urls, options);
-    if (robotsTxtFile) {
-        const skippedRequests = [];
-        requestOptions = requestOptions.filter((request) => {
-            if (robotsTxtFile.isAllowed(request.url)) {
-                return true;
-            }
-            skippedRequests.push(request);
-            return false;
-        });
-        await reportSkippedRequests(skippedRequests, 'robotsTxt');
-    }
-    if (transformRequestFunction) {
-        const skippedRequests = [];
-        requestOptions = requestOptions
-            .map((request) => {
-            const transformedRequest = transformRequestFunction(request);
-            if (!transformedRequest) {
-                skippedRequests.push(request);
-            }
-            return transformedRequest;
-        })
-            .filter((r) => Boolean(r));
-        await reportSkippedRequests(skippedRequests, 'filters');
-    }
-    async function createFilteredRequests() {
-        const skippedRequests = [];
-        // No user provided patterns means we can skip an extra filtering step
-        if (urlPatternObjects.length === 0) {
-            return createRequests(requestOptions, enqueueStrategyPatterns, urlExcludePatternObjects, options.strategy, (url) => skippedRequests.push(url));
-        }
-        // Generate requests based on the user patterns first
-        const generatedRequestsFromUserFilters = createRequests(requestOptions, urlPatternObjects, urlExcludePatternObjects, options.strategy, (url) => skippedRequests.push(url));
-        // ...then filter them by the enqueue links strategy (making this an AND check)
-        const filtered = filterRequestsByPatterns(generatedRequestsFromUserFilters, enqueueStrategyPatterns, (url) => skippedRequests.push(url));
-        await reportSkippedRequests(skippedRequests.map((url) => ({ url })), 'filters');
-        return filtered;
-    }
-    let requests = await createFilteredRequests();
-    if (typeof limit === 'number' && limit < requests.length) {
-        await reportSkippedRequests(requests.slice(limit), 'enqueueLimit');
-        requests = requests.slice(0, limit);
-    }
-    const { addedRequests } = await requestQueue.addRequestsBatched(requests, {
-        forefront,
-        waitForAllRequestsToBeAdded,
-    });
-    return { processedRequests: addedRequests, unprocessedRequests: [] };
-}
+export { EnqueueStrategy };
 /**
  * @internal
  * This method helps resolve the baseUrl that will be used for filtering in {@link enqueueLinks}.
@@ -246,24 +21,57 @@ export function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy, finalR
         return finalUrlOrigin;
     }
     // If the user wants to ensure the same domain is accessed, regardless of subdomains, we check to ensure the domains match
-    // Returning undefined here is intentional! If the domains don't match, having no baseUrl in enqueueLinks will cause it to not enqueue anything
-    // which is the intended behavior (since we went off domain)
+    // If they don't (we went off domain via a redirect), we keep filtering against the original domain - returning undefined would disable the filtering entirely
     if (enqueueStrategy === EnqueueStrategy.SameDomain) {
         const originalHostname = getDomain(originalUrlOrigin, { mixedInputs: false });
         const finalHostname = getDomain(finalUrlOrigin, { mixedInputs: false });
         if (originalHostname === finalHostname) {
             return finalUrlOrigin;
         }
-        return undefined;
+        return originalUrlOrigin;
     }
     // Always enqueue urls that are from the same origin in all other cases, as the filtering happens on the original request url, even if there was a redirect
     // before actually finding the urls
     return originalUrlOrigin;
 }
 /**
- * Internal function that changes the enqueue globs to match both http and https
+ * @internal
+ * Builds the glob patterns a URL must match to satisfy the given enqueue `strategy`, anchored at `baseUrl`.
+ */
+export function buildEnqueueStrategyPatterns(baseUrl, strategy) {
+    const url = new URL(baseUrl);
+    switch (strategy) {
+        case EnqueueStrategy.SameHostname:
+            // We need to get the origin of the passed in domain in the event someone sets baseUrl
+            // to an url like https://example.com/deep/default/path and one of the found urls is an
+            // absolute relative path (/path/to/page)
+            return [{ glob: ignoreHttpSchema(`${url.origin}/**`) }];
+        case EnqueueStrategy.SameDomain: {
+            // Get the actual hostname from the base url
+            const baseUrlHostname = getDomain(url.hostname, { mixedInputs: false });
+            if (baseUrlHostname) {
+                // We have a hostname, so we can use it to match all links on the page that point to it and any subdomains of it
+                url.hostname = baseUrlHostname;
+                return [
+                    { glob: ignoreHttpSchema(`${url.origin.replace(baseUrlHostname, `*.${baseUrlHostname}`)}/**`) },
+                    { glob: ignoreHttpSchema(`${url.origin}/**`) },
+                ];
+            }
+            // We don't have a hostname (can happen for ips for instance), so reproduce the same behavior
+            // as SameDomainAndSubdomain
+            return [{ glob: ignoreHttpSchema(`${url.origin}/**`) }];
+        }
+        case EnqueueStrategy.SameOrigin:
+            // The same behavior as SameHostname, but respecting the protocol of the URL
+            return [{ glob: `${url.origin}/**` }];
+        case EnqueueStrategy.All:
+        default:
+            return [{ glob: `http{s,}://**` }];
+    }
+}
+/**
+ * Internal function that changes the enqueue glob patterns to match both http and https
  */
 function ignoreHttpSchema(pattern) {
     return pattern.replace(/^(https?):\/\//, 'http{s,}://');
 }
-//# sourceMappingURL=enqueue_links.js.map

package/enqueue_links/index.d.ts

@@ -1,3 +1,2 @@
 export * from './enqueue_links.js';
 export * from './shared.js';
-//# sourceMappingURL=index.d.ts.map

package/enqueue_links/index.js

@@ -1,3 +1,2 @@
 export * from './enqueue_links.js';
 export * from './shared.js';
-//# sourceMappingURL=index.js.map

package/enqueue_links/shared.d.ts

@@ -1,39 +1,43 @@
-import type { Awaitable } from '@crawlee/types';
-import type { RequestOptions } from '../request.js';
+import type { Awaitable, Dictionary } from '@crawlee/types';
+import { z } from 'zod';
+import type { RequestOptions, Source } from '../request.js';
 import { Request } from '../request.js';
-import type { EnqueueLinksOptions } from './enqueue_links.js';
-export { tryAbsoluteURL } from '@crawlee/utils';
-export type UrlPatternObject = {
+import type { EnqueueStrategyOption } from './enqueue_links.js';
+export interface UrlPatternObject {
     glob?: string;
     regexp?: RegExp;
-} & Pick<RequestOptions, 'method' | 'payload' | 'label' | 'userData' | 'headers'>;
-export type PseudoUrlObject = {
-    purl: string;
-} & Pick<RequestOptions, 'method' | 'payload' | 'label' | 'userData' | 'headers'>;
-export type PseudoUrlInput = string | PseudoUrlObject;
-export type GlobObject = {
+}
+export interface GlobObject {
     glob: string;
-} & Pick<RequestOptions, 'method' | 'payload' | 'label' | 'userData' | 'headers'>;
+}
 export type GlobInput = string | GlobObject;
-export type RegExpObject = {
+export interface RegExpObject {
     regexp: RegExp;
-} & Pick<RequestOptions, 'method' | 'payload' | 'label' | 'userData' | 'headers'>;
+}
 export type RegExpInput = RegExp | RegExpObject;
-export type SkippedRequestReason = 'robotsTxt' | 'limit' | 'enqueueLimit' | 'filters' | 'redirect' | 'depth';
+/** Unified URL pattern input — accepts glob strings, glob objects, RegExp instances, or regexp objects. */
+export type UrlPatternInput = GlobInput | RegExpInput;
+/**
+ * Accepts one {@link UrlPatternInput} — a glob string, a RegExp instance, or a `{ glob }` / `{ regexp }` object.
+ * @internal
+ */
+export declare const urlPatternSchema: z.ZodType<UrlPatternInput>;
+export type SkippedRequestReason = 'robotsTxt' | 'limit' | 'enqueueLimit' | 'filters' | 'transform' | 'redirect' | 'depth';
 export type SkippedRequestCallback = (args: {
-    url: string;
+    request: Request;
     reason: SkippedRequestReason;
 }) => Awaitable<void>;
 /**
+ * Builds the `{ request, reason }` argument passed to a {@link SkippedRequestCallback}, constructing the
+ * `Request` lazily on first access to `request` (and caching it) since most skips are never observed by a
+ * real callback and building a `Request` isn't free.
  * @ignore
  */
-export declare function updateEnqueueLinksPatternCache(item: GlobInput | RegExpInput | PseudoUrlInput, pattern: RegExpObject | GlobObject): void;
+export declare function createSkippedRequestArgs(source: string | Source, reason: SkippedRequestReason): Parameters<SkippedRequestCallback>[0];
 /**
- * Helper factory used in the `enqueueLinks()` and enqueueLinksByClickingElements() function
- * to construct RegExps from PseudoUrl strings.
  * @ignore
  */
-export declare function constructRegExpObjectsFromPseudoUrls(pseudoUrls: readonly PseudoUrlInput[]): RegExpObject[];
+export declare function updateEnqueueLinksPatternCache(item: GlobInput | RegExpInput, pattern: RegExpObject | GlobObject): void;
 /**
  * Helper factory used in the `enqueueLinks()` and enqueueLinksByClickingElements() function
  * to construct Glob objects from Glob pattern strings.
@@ -51,23 +55,46 @@ export declare function validateGlobPattern(glob: string): string;
  */
 export declare function constructRegExpObjectsFromRegExps(regexps: readonly RegExpInput[]): RegExpObject[];
 /**
+ * Helper factory used in the `enqueueLinks()` function to construct UrlPatternObjects
+ * from a mixed array of glob strings, glob objects, RegExp instances, and regexp objects.
  * @ignore
  */
-export declare function createRequests(requestOptions: (string | RequestOptions)[], urlPatternObjects?: UrlPatternObject[], excludePatternObjects?: UrlPatternObject[], strategy?: EnqueueLinksOptions['strategy'], onSkippedUrl?: (url: string) => void): Request[];
-export declare function filterRequestsByPatterns(requests: Request[], patterns?: UrlPatternObject[], onSkippedUrl?: (url: string) => void): Request[];
+export declare function constructUrlPatternObjects(patterns: readonly UrlPatternInput[]): UrlPatternObject[];
 /**
+ * Filters request options by URL patterns.
+ *
+ * When `includePatterns` is empty/undefined, all options pass through (only exclude filtering applies).
  * @ignore
  */
-export declare function createRequestOptions(sources: readonly (string | Record<string, unknown>)[], options?: Pick<EnqueueLinksOptions, 'label' | 'userData' | 'baseUrl' | 'skipNavigation' | 'strategy'>): RequestOptions[];
+export declare function filterRequestOptionsByPatterns(requestOptions: RequestOptions[], includePatterns: UrlPatternObject[] | undefined, excludePatterns?: UrlPatternObject[], strategy?: EnqueueStrategyOption, onSkippedRequestOptions?: (options: RequestOptions) => void): RequestOptions[];
 /**
- * Takes an Apify {@link RequestOptions} object and changes its attributes in a desired way. This user-function is used
- * {@link enqueueLinks} to modify requests before enqueuing them.
+ * @ignore
+ */
+export declare function createRequestOptions(sources: readonly (string | Record<string, unknown>)[], options?: {
+    label?: string;
+    userData?: Dictionary;
+    baseUrl?: string;
+    skipNavigation?: boolean;
+    sessionId?: string;
+    strategy?: EnqueueStrategyOption;
+}): RequestOptions[];
+/**
+ * Takes a {@link RequestOptions} object and changes its attributes in a desired way. This user-function is used
+ * by {@link enqueueLinks} to modify request options before they are converted to {@link Request} instances.
  */
 export interface RequestTransform {
     /**
      * @param original Request options to be modified.
-     * @returns The modified request options to enqueue.
+     * @returns The modified request options to enqueue, `'unchanged'` to keep the original options as-is,
+     *   or a falsy value / `'skip'` to exclude the request from the queue.
      */
-    (original: RequestOptions): RequestOptions | false | undefined | null;
+    (original: RequestOptions): RequestOptions | false | undefined | null | 'skip' | 'unchanged';
 }
-//# sourceMappingURL=shared.d.ts.map
+/**
+ * Applies a {@link RequestTransform} function to a list of request options.
+ * Options for which the transform returns a falsy value are removed from the list.
+ * @param onSkipped Called with the original request options when the transform returns a falsy value (i.e. the request is skipped).
+ * @ignore
+ * @internal
+ */
+export declare function applyRequestTransform(requestOptions: RequestOptions[], transformFn: RequestTransform, onSkipped?: (requestOptions: RequestOptions) => void): RequestOptions[];